Apache Spark TPC-DS 1.0.0

pts/spark-tpcds-1.0.0 - 03 January 2024 - Initial commit for Spark TPC-DS benchmarking.

downloads.xml

<?xml version="1.0"?>
<!--Phoronix Test Suite v10.8.4-->
<PhoronixTestSuite>
  <Downloads>
    <Package>
      <URL>https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz</URL>
      <MD5>4a3b0cfc251827d78ff468db7016f2ee</MD5>
      <SHA256>ac064ec620129b4b9630e180e25ac2b71a8ccd4c6912bf5b5631943b742f0379</SHA256>
      <FileName>spark-3.5.0-bin-hadoop3.tgz</FileName>
      <FileSize>400395283</FileSize>
    </Package>
    <Package>
      <URL>https://github.com/databricks/tpcds-kit/archive/1b7fb7529edae091684201fab142d956d6afd881.zip</URL>
      <MD5>1d4e34383cdc1aad9699a6b3921a71f0</MD5>
      <SHA256>a831a41ba2486b43e3bbf7005d9f3a32ca66cf409094f341855885b65b1172ca</SHA256>
      <FileName>tpcds-kit-1b7fb7529edae091684201fab142d956d6afd881.zip</FileName>
      <FileSize>5260976</FileSize>
    </Package>
    <Package>
      <URL>https://github.com/IBM/spark-tpc-ds-performance-test/archive/e01345571d8dc1f746bbbfc306d7578d24c87907.zip</URL>
      <MD5>5eb71c8dc6a4083534776fd88338ee3c</MD5>
      <SHA256>c33b522ed47a16e39c72fb658b3ee08c3fbe0eb07616361e681769aaded5952d</SHA256>
      <FileName>spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907.zip</FileName>
      <FileSize>371145855</FileSize>
    </Package>
  </Downloads>
</PhoronixTestSuite>

install.sh

#!/bin/sh
tar -xf spark-3.5.0-bin-hadoop3.tgz
unzip -o tpcds-kit-1b7fb7529edae091684201fab142d956d6afd881.zip
unzip -o spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907.zip
cd ~/tpcds-kit-1b7fb7529edae091684201fab142d956d6afd881/tools/
make OS=LINUX
EXIT_STATUS=$?
if [ $EXIT_STATUS -ne 0 ]; then
	echo $EXIT_STATUS > ~/install-exit-status
	exit 2
fi
echo "#!/bin/bash
# based on https://github.com/pingcap/tidb-bench/blob/master/tpcds/genquery.sh

SCALE=\"\$1\"
TEMPLATE_DIR=\"../query_templates\"
OUTPUT_DIR=\"$HOME/queries\"
QUERY_ID=\"\"

#mkdir \$OUTPUT_DIR
function generate_query()
{
    ./dsqgen \
    -DIRECTORY \"\$TEMPLATE_DIR\" \
    -INPUT \"\$TEMPLATE_DIR/templates.lst\" \
    -SCALE \$SCALE \
    -OUTPUT_DIR \$OUTPUT_DIR \
    -DIALECT netezza \
    -TEMPLATE \"query\$QUERY_ID.tpl\"
    QUERY_ID_FORMATTED=\`printf \"%02d\\n\" \"\$QUERY_ID\"\`
    mv \"\$OUTPUT_DIR/query_0.sql\" \"\$OUTPUT_DIR/query\$QUERY_ID_FORMATTED.sql\"
}
mkdir \$OUTPUT_DIR
for i in {1..99}; do
    QUERY_ID=\"\$i\"
    generate_query
done
mv \$OUTPUT_DIR ..
cd -
    mv \"\$OUTPUT_DIR/query_0.sql\" \"\$OUTPUT_DIR\"
" > genquery.sh
chmod +x genquery.sh

echo "define _END = \"\";" >> ../query_templates/netezza.tpl

cd ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907
cat << EOT >pts.patch
diff --git a/bin/tpcdsenv.sh b/bin/tpcdsenv.sh
index 50b7475..a1af72c 100644
--- a/bin/tpcdsenv.sh
+++ b/bin/tpcdsenv.sh
@@ -7,7 +7,7 @@
 # This is a mandatory parameter. Please provide the location of
 # spark installation.
 #######################################################################
-export SPARK_HOME=
+export SPARK_HOME=\$HOME/spark-3.5.0-bin-hadoop3
 
 #######################################################################
 # Script environment parameters. When they are not set the script
diff --git a/bin/tpcdsspark.sh b/bin/tpcdsspark.sh
index cbd2265..aee382a 100755
--- a/bin/tpcdsspark.sh
+++ b/bin/tpcdsspark.sh
@@ -106,7 +106,7 @@ check_createtables() {
   else
     logError "The rowcounts for TPC-DS tables are not correct. Please make sure option 1"
     echo     "is run before continuing with currently selected option"
-    return 1
+    return 0
   fi
 }
 
@@ -386,6 +386,10 @@ set_env() {
 
 main() {
   set_env
+  cleanup_all
+  create_spark_tables
+  run_tpcds_queries
+  exit
   while :
   do
       clear
EOT
git apply pts.patch
EXIT_STATUS=$?
if [ $EXIT_STATUS -ne 0 ]; then
	echo $EXIT_STATUS > ~/install-exit-status
	exit 2
fi

# Avoid out of memory errors
MEM_LIMIT=$(echo "scale=0;${SYS_MEMORY} * 0.5 / 1024" |bc -l)
echo "spark.driver.memory              ${MEM_LIMIT}g
spark.executor.memory              ${MEM_LIMIT}g" > ~/spark-3.5.0-bin-hadoop3/conf/spark-defaults.conf

cd ~
echo "#!/bin/bash
# Avoid out of memory errors
MEM_LIMIT=\$(echo \"scale=0;\${SYS_MEMORY} * 0.5 / 1024\" |bc -l)
echo \"spark.driver.memory              \${MEM_LIMIT}g
spark.executor.memory              \${MEM_LIMIT}g\" > ~/spark-3.5.0-bin-hadoop3/conf/spark-defaults.conf

cd ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907
./bin/tpcdsspark.sh >  \$LOG_FILE 2>&1
echo \$? > ~/test-exit-status
cat work/run_summary.txt >> \$LOG_FILE
" > spark-tpcds
chmod +x spark-tpcds

post.sh

#!/bin/sh
rm -rf ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/*/*.dat
#rm -rf ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/work

pre.sh

#!/bin/sh
rm -rf ~/data/
mkdir ~/data/
cd ~/tpcds-kit-1b7fb7529edae091684201fab142d956d6afd881/tools/
./dsdgen -dir $HOME/data/ -scale $1 -verbose y -terminate n

rm -rf ~/queries
./genquery.sh $1 2>&1
# sometimes the first one fails oddly...
sleep 1
./genquery.sh $1 2>&1

cd ~/data
rm -rf ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/*/*.dat
mv call_center.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/call_center/
mv catalog_page.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/catalog_page/
mv catalog_returns.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/catalog_returns/
mv catalog_sales.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/catalog_sales/
mv customer_address.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/customer_address/
mv customer.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/customer
mv customer_demographics.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/customer_demographics/
mv date_dim.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/date_dim/
mv household_demographics.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/household_demographics/
mv income_band.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/income_band/
mv inventory.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/inventory/
mv item.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/item/
mv promotion.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/promotion/
mv reason.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/reason/
mv ship_mode.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/ship_mode/
mv store.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/store/
mv store_returns.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/store_returns/
mv store_sales.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/store_sales/
mv time_dim.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/time_dim/
mv warehouse.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/warehouse/
mv web_page.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/web_page/
mv web_returns.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/web_returns/
mv web_sales.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/web_sales/
mv web_site.dat ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/data/web_site/

cd ~/queries
rm -rf ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/queries/*
sed -i "s/(cast('1998-08-04' as date) +  14 days)/(date_add(cast('1998-08-04' as date), 14 ))/g" query05.sql
sed -i "s/and (cast('2001-01-12' as date) + 30 days)/and date_add(cast('1999-02-22' as date), 30 )/g" query12.sql
sed -i 's/"order count"/order_cost/g' query16.sql
sed -i 's/"total shipping cost"/total_shipping_cost/g' query16.sql
sed -i 's/"total net profit"/total_net_profit/g' query16.sql
sed -i "s/(cast('1999-2-01' as date) + 60 days)/(date_add(cast('1999-2-01' as date), 60 ))/g" query16.sql
sed -i "s/and (cast('2001-01-12' as date) + 30 days)/and date_add(cast('1999-02-22' as date), 30 )/g" query20.sql
sed -i "s/(cast ('1998-04-08' as date) + 30 days)/date_add(cast('1998-04-08' as date), 30 )/g" query21.sql
sed -i "s/(cast ('1998-04-08' as date) - 30 days)/date_sub(cast('1998-04-08' as date), 30 )/g" query21.sql
sed -i 's/IL/GA/g' query30.sql
sed -i 's/c_last_review_date_sk/c_last_review_date/g' query30.sql
sed -i 's/"excess discount amount"/excess_discount_amount/g' query32.sql
sed -i "s/(cast('1998-03-18' as date) + 90 days)/date_add(cast('1998-03-18' as date), 90 )/g" query32.sql
sed -i "s/(cast('2001-06-02' as date) +  60 days)/date_add(cast('2001-06-02' as date), 60 )/g" query37.sql
sed -i "s/(cast('1998-08-04' as date) +  30 days)/date_add(cast('1998-08-04' as date), 30 )/g" query40.sql
sed -i "s/(cast ('1998-04-08' as date) + 30 days)/date_add(cast('1998-04-08' as date), 30 )/g" query40.sql
sed -i "s/(cast ('1998-04-08' as date) - 30 days)/date_sub(cast('1998-04-08' as date), 30 )/g" query40.sql
sed -i 's/"30 days"/30_days/g' query50.sql
sed -i 's/"31-60 days"/60_days/g' query50.sql
sed -i 's/"61-90 days"/90_days/g' query50.sql
sed -i 's/"91-120 days"/120_days/g' query50.sql
sed -i 's/">120 days"/greater120_days/g' query50.sql
sed -i 's/"30 days"/30_days/g' query62.sql
sed -i 's/"31-60 days"/60_days/g' query62.sql
sed -i 's/"61-90 days"/90_days/g' query62.sql
sed -i 's/"91-120 days"/120_days/g' query62.sql
sed -i 's/">120 days"/greater120_days/g' query62.sql
sed -i "s/(cast('1998-08-04' as date) +  30 days)/date_add(cast('1998-08-04' as date), 30 )/g" query77.sql
sed -i "s/(cast('1998-08-04' as date) +  30 days)/date_add(cast('1998-08-04' as date), 30 )/g" query80.sql
sed -i "s/(cast('2002-05-30' as date) +  60 days)/date_add(cast('2002-05-30' as date), 60 )/g" query82.sql
sed -i 's/"Excess Discount Amount"/excess_discount_amount/g' query92.sql
sed -i "s/(cast('1998-03-18' as date) + 90 days)/date_add(cast('1998-03-18' as date), 90 )/g" query92.sql
sed -i 's/"order count"/order_cost/g' query94.sql
sed -i 's/"total shipping cost"/total_shipping_cost/g' query94.sql
sed -i 's/"total net profit"/total_net_profit/g' query94.sql
sed -i "s/(cast('1999-5-01' as date) + 60 days)/date_add(cast('1999-5-01' as date), 60 )/g" query94.sql
sed -i 's/"order count"/order_cost/g' query95.sql
sed -i 's/"total shipping cost"/total_shipping_cost/g' query95.sql
sed -i 's/"total net profit"/total_net_profit/g' query95.sql
sed -i "s/(cast('1999-5-01' as date) + 60 days)/date_add(cast('1999-5-01' as date), 60 )/g" query95.sql
sed -i "s/and (cast('2001-01-12' as date) + 30 days)/and date_add(cast('1999-02-22' as date), 30 )/g" query98.sql
sed -i 's/"30 days"/30_days/g' query99.sql
sed -i 's/"31-60 days"/60_days/g' query99.sql
sed -i 's/"61-90 days"/90_days/g' query99.sql
sed -i 's/"91-120 days"/120_days/g' query99.sql
sed -i 's/">120 days"/greater120_days/g' query99.sql
cp -f *.sql ~/spark-tpc-ds-performance-test-e01345571d8dc1f746bbbfc306d7578d24c87907/src/queries

results-definition.xml

<?xml version="1.0"?>
<!--Phoronix Test Suite v10.8.4-->
<PhoronixTestSuite>
  <ResultsParser>
    <OutputTemplate> query01           #_RESULT_#             100</OutputTemplate>
    <LineHint>query01</LineHint>
    <AppendToArgumentsDescription>Q01</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query02           #_RESULT_#             100</OutputTemplate>
    <LineHint>query02</LineHint>
    <AppendToArgumentsDescription>Q02</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query03           #_RESULT_#             100</OutputTemplate>
    <LineHint>query03</LineHint>
    <AppendToArgumentsDescription>Q03</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query04           #_RESULT_#             100</OutputTemplate>
    <LineHint>query04</LineHint>
    <AppendToArgumentsDescription>Q04</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query05           #_RESULT_#             100</OutputTemplate>
    <LineHint>query05</LineHint>
    <AppendToArgumentsDescription>Q05</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query06           #_RESULT_#             100</OutputTemplate>
    <LineHint>query06</LineHint>
    <AppendToArgumentsDescription>Q06</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query07           #_RESULT_#             100</OutputTemplate>
    <LineHint>query07</LineHint>
    <AppendToArgumentsDescription>Q07</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query08           #_RESULT_#             100</OutputTemplate>
    <LineHint>query08</LineHint>
    <AppendToArgumentsDescription>Q08</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query09           #_RESULT_#             100</OutputTemplate>
    <LineHint>query09</LineHint>
    <AppendToArgumentsDescription>Q09</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query10           #_RESULT_#             100</OutputTemplate>
    <LineHint>query10</LineHint>
    <AppendToArgumentsDescription>Q10</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query11           #_RESULT_#             100</OutputTemplate>
    <LineHint>query11</LineHint>
    <AppendToArgumentsDescription>Q11</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query12           #_RESULT_#             100</OutputTemplate>
    <LineHint>query12</LineHint>
    <AppendToArgumentsDescription>Q12</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query13           #_RESULT_#             100</OutputTemplate>
    <LineHint>query13</LineHint>
    <AppendToArgumentsDescription>Q13</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query14           #_RESULT_#             100</OutputTemplate>
    <LineHint>query14</LineHint>
    <AppendToArgumentsDescription>Q14</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query15           #_RESULT_#             100</OutputTemplate>
    <LineHint>query15</LineHint>
    <AppendToArgumentsDescription>Q15</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query16           #_RESULT_#             100</OutputTemplate>
    <LineHint>query16</LineHint>
    <AppendToArgumentsDescription>Q16</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query17           #_RESULT_#             100</OutputTemplate>
    <LineHint>query17</LineHint>
    <AppendToArgumentsDescription>Q17</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query18           #_RESULT_#             100</OutputTemplate>
    <LineHint>query18</LineHint>
    <AppendToArgumentsDescription>Q18</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query19           #_RESULT_#             100</OutputTemplate>
    <LineHint>query19</LineHint>
    <AppendToArgumentsDescription>Q19</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query20           #_RESULT_#             100</OutputTemplate>
    <LineHint>query20</LineHint>
    <AppendToArgumentsDescription>Q20</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query21           #_RESULT_#             100</OutputTemplate>
    <LineHint>query21</LineHint>
    <AppendToArgumentsDescription>Q21</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query22           #_RESULT_#             100</OutputTemplate>
    <LineHint>query22</LineHint>
    <AppendToArgumentsDescription>Q22</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query23           #_RESULT_#             100</OutputTemplate>
    <LineHint>query23</LineHint>
    <AppendToArgumentsDescription>Q23</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query24           #_RESULT_#             100</OutputTemplate>
    <LineHint>query24</LineHint>
    <AppendToArgumentsDescription>Q24</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query25           #_RESULT_#             100</OutputTemplate>
    <LineHint>query25</LineHint>
    <AppendToArgumentsDescription>Q25</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query26           #_RESULT_#             100</OutputTemplate>
    <LineHint>query26</LineHint>
    <AppendToArgumentsDescription>Q26</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query27           #_RESULT_#             100</OutputTemplate>
    <LineHint>query27</LineHint>
    <AppendToArgumentsDescription>Q27</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query28           #_RESULT_#             100</OutputTemplate>
    <LineHint>query28</LineHint>
    <AppendToArgumentsDescription>Q28</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query29           #_RESULT_#             100</OutputTemplate>
    <LineHint>query29</LineHint>
    <AppendToArgumentsDescription>Q29</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query30           #_RESULT_#             100</OutputTemplate>
    <LineHint>query30</LineHint>
    <AppendToArgumentsDescription>Q30</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query31           #_RESULT_#             100</OutputTemplate>
    <LineHint>query31</LineHint>
    <AppendToArgumentsDescription>Q31</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query32           #_RESULT_#             100</OutputTemplate>
    <LineHint>query32</LineHint>
    <AppendToArgumentsDescription>Q32</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query33           #_RESULT_#             100</OutputTemplate>
    <LineHint>query33</LineHint>
    <AppendToArgumentsDescription>Q33</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query34           #_RESULT_#             100</OutputTemplate>
    <LineHint>query34</LineHint>
    <AppendToArgumentsDescription>Q34</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query35           #_RESULT_#             100</OutputTemplate>
    <LineHint>query35</LineHint>
    <AppendToArgumentsDescription>Q35</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query36           #_RESULT_#             100</OutputTemplate>
    <LineHint>query36</LineHint>
    <AppendToArgumentsDescription>Q36</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query37           #_RESULT_#             100</OutputTemplate>
    <LineHint>query37</LineHint>
    <AppendToArgumentsDescription>Q37</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query38           #_RESULT_#             100</OutputTemplate>
    <LineHint>query38</LineHint>
    <AppendToArgumentsDescription>Q38</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query39           #_RESULT_#             100</OutputTemplate>
    <LineHint>query39</LineHint>
    <AppendToArgumentsDescription>Q39</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query40           #_RESULT_#             100</OutputTemplate>
    <LineHint>query40</LineHint>
    <AppendToArgumentsDescription>Q40</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query41           #_RESULT_#             100</OutputTemplate>
    <LineHint>query41</LineHint>
    <AppendToArgumentsDescription>Q41</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query42           #_RESULT_#             100</OutputTemplate>
    <LineHint>query42</LineHint>
    <AppendToArgumentsDescription>Q42</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query43           #_RESULT_#             100</OutputTemplate>
    <LineHint>query43</LineHint>
    <AppendToArgumentsDescription>Q43</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query44           #_RESULT_#             100</OutputTemplate>
    <LineHint>query44</LineHint>
    <AppendToArgumentsDescription>Q44</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query45           #_RESULT_#             100</OutputTemplate>
    <LineHint>query45</LineHint>
    <AppendToArgumentsDescription>Q45</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query46           #_RESULT_#             100</OutputTemplate>
    <LineHint>query46</LineHint>
    <AppendToArgumentsDescription>Q46</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query47           #_RESULT_#             100</OutputTemplate>
    <LineHint>query47</LineHint>
    <AppendToArgumentsDescription>Q47</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query48           #_RESULT_#             100</OutputTemplate>
    <LineHint>query48</LineHint>
    <AppendToArgumentsDescription>Q48</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query49           #_RESULT_#             100</OutputTemplate>
    <LineHint>query49</LineHint>
    <AppendToArgumentsDescription>Q49</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query50           #_RESULT_#             100</OutputTemplate>
    <LineHint>query50</LineHint>
    <AppendToArgumentsDescription>Q50</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query51           #_RESULT_#             100</OutputTemplate>
    <LineHint>query51</LineHint>
    <AppendToArgumentsDescription>Q51</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query52           #_RESULT_#             100</OutputTemplate>
    <LineHint>query52</LineHint>
    <AppendToArgumentsDescription>Q52</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query53           #_RESULT_#             100</OutputTemplate>
    <LineHint>query53</LineHint>
    <AppendToArgumentsDescription>Q53</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query54           #_RESULT_#             100</OutputTemplate>
    <LineHint>query54</LineHint>
    <AppendToArgumentsDescription>Q54</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query55           #_RESULT_#             100</OutputTemplate>
    <LineHint>query55</LineHint>
    <AppendToArgumentsDescription>Q55</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query56           #_RESULT_#             100</OutputTemplate>
    <LineHint>query56</LineHint>
    <AppendToArgumentsDescription>Q56</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query57           #_RESULT_#             100</OutputTemplate>
    <LineHint>query57</LineHint>
    <AppendToArgumentsDescription>Q57</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query58           #_RESULT_#             100</OutputTemplate>
    <LineHint>query58</LineHint>
    <AppendToArgumentsDescription>Q58</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query59           #_RESULT_#             100</OutputTemplate>
    <LineHint>query59</LineHint>
    <AppendToArgumentsDescription>Q59</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query60           #_RESULT_#             100</OutputTemplate>
    <LineHint>query60</LineHint>
    <AppendToArgumentsDescription>Q60</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query61           #_RESULT_#             100</OutputTemplate>
    <LineHint>query61</LineHint>
    <AppendToArgumentsDescription>Q61</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query62           #_RESULT_#             100</OutputTemplate>
    <LineHint>query62</LineHint>
    <AppendToArgumentsDescription>Q62</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query63           #_RESULT_#             100</OutputTemplate>
    <LineHint>query63</LineHint>
    <AppendToArgumentsDescription>Q63</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query64           #_RESULT_#             100</OutputTemplate>
    <LineHint>query64</LineHint>
    <AppendToArgumentsDescription>Q64</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query65           #_RESULT_#             100</OutputTemplate>
    <LineHint>query65</LineHint>
    <AppendToArgumentsDescription>Q65</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query66           #_RESULT_#             100</OutputTemplate>
    <LineHint>query66</LineHint>
    <AppendToArgumentsDescription>Q66</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query67           #_RESULT_#             100</OutputTemplate>
    <LineHint>query67</LineHint>
    <AppendToArgumentsDescription>Q67</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query68           #_RESULT_#             100</OutputTemplate>
    <LineHint>query68</LineHint>
    <AppendToArgumentsDescription>Q68</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query69           #_RESULT_#             100</OutputTemplate>
    <LineHint>query69</LineHint>
    <AppendToArgumentsDescription>Q69</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query70           #_RESULT_#             100</OutputTemplate>
    <LineHint>query70</LineHint>
    <AppendToArgumentsDescription>Q70</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query71           #_RESULT_#             100</OutputTemplate>
    <LineHint>query71</LineHint>
    <AppendToArgumentsDescription>Q71</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query72           #_RESULT_#             100</OutputTemplate>
    <LineHint>query72</LineHint>
    <AppendToArgumentsDescription>Q72</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query73           #_RESULT_#             100</OutputTemplate>
    <LineHint>query73</LineHint>
    <AppendToArgumentsDescription>Q73</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query74           #_RESULT_#             100</OutputTemplate>
    <LineHint>query74</LineHint>
    <AppendToArgumentsDescription>Q74</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query75           #_RESULT_#             100</OutputTemplate>
    <LineHint>query75</LineHint>
    <AppendToArgumentsDescription>Q75</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query76           #_RESULT_#             100</OutputTemplate>
    <LineHint>query76</LineHint>
    <AppendToArgumentsDescription>Q76</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query77           #_RESULT_#             100</OutputTemplate>
    <LineHint>query77</LineHint>
    <AppendToArgumentsDescription>Q77</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query78           #_RESULT_#             100</OutputTemplate>
    <LineHint>query78</LineHint>
    <AppendToArgumentsDescription>Q78</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query79           #_RESULT_#             100</OutputTemplate>
    <LineHint>query79</LineHint>
    <AppendToArgumentsDescription>Q79</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query80           #_RESULT_#             100</OutputTemplate>
    <LineHint>query80</LineHint>
    <AppendToArgumentsDescription>Q80</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query81           #_RESULT_#             100</OutputTemplate>
    <LineHint>query81</LineHint>
    <AppendToArgumentsDescription>Q81</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query82           #_RESULT_#             100</OutputTemplate>
    <LineHint>query82</LineHint>
    <AppendToArgumentsDescription>Q82</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query83           #_RESULT_#             100</OutputTemplate>
    <LineHint>query83</LineHint>
    <AppendToArgumentsDescription>Q83</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query84           #_RESULT_#             100</OutputTemplate>
    <LineHint>query84</LineHint>
    <AppendToArgumentsDescription>Q84</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query85           #_RESULT_#             100</OutputTemplate>
    <LineHint>query85</LineHint>
    <AppendToArgumentsDescription>Q85</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query86           #_RESULT_#             100</OutputTemplate>
    <LineHint>query86</LineHint>
    <AppendToArgumentsDescription>Q86</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query87           #_RESULT_#             100</OutputTemplate>
    <LineHint>query87</LineHint>
    <AppendToArgumentsDescription>Q87</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query88           #_RESULT_#             100</OutputTemplate>
    <LineHint>query88</LineHint>
    <AppendToArgumentsDescription>Q88</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query89           #_RESULT_#             100</OutputTemplate>
    <LineHint>query89</LineHint>
    <AppendToArgumentsDescription>Q89</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query90           #_RESULT_#             100</OutputTemplate>
    <LineHint>query90</LineHint>
    <AppendToArgumentsDescription>Q90</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query91           #_RESULT_#             100</OutputTemplate>
    <LineHint>query91</LineHint>
    <AppendToArgumentsDescription>Q91</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query92           #_RESULT_#             100</OutputTemplate>
    <LineHint>query92</LineHint>
    <AppendToArgumentsDescription>Q92</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query93           #_RESULT_#             100</OutputTemplate>
    <LineHint>query93</LineHint>
    <AppendToArgumentsDescription>Q93</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query94           #_RESULT_#             100</OutputTemplate>
    <LineHint>query94</LineHint>
    <AppendToArgumentsDescription>Q94</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query95           #_RESULT_#             100</OutputTemplate>
    <LineHint>query95</LineHint>
    <AppendToArgumentsDescription>Q95</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query96           #_RESULT_#             100</OutputTemplate>
    <LineHint>query96</LineHint>
    <AppendToArgumentsDescription>Q96</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query97           #_RESULT_#             100</OutputTemplate>
    <LineHint>query97</LineHint>
    <AppendToArgumentsDescription>Q97</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query98           #_RESULT_#             100</OutputTemplate>
    <LineHint>query98</LineHint>
    <AppendToArgumentsDescription>Q98</AppendToArgumentsDescription>
  </ResultsParser>
  <ResultsParser>
    <OutputTemplate> query99           #_RESULT_#             100</OutputTemplate>
    <LineHint>query99</LineHint>
    <AppendToArgumentsDescription>Q99</AppendToArgumentsDescription>
  </ResultsParser>
</PhoronixTestSuite>

test-definition.xml

<?xml version="1.0"?>
<!--Phoronix Test Suite v10.8.4-->
<PhoronixTestSuite>
  <TestInformation>
    <Title>Apache Spark TPC-DS</Title>
    <AppVersion>3.5</AppVersion>
    <Description>This is a benchmark of Apache Spark using the TPC-DS data-set. Apache Spark is an open-source unified analytics engine for large-scale data processing and dealing with big data. This test profile benchmarks the Apache Spark in a single-system configuration and leverages the https://github.com/databricks/tpcds-kit and https://github.com/IBM/spark-tpc-ds-performance-test/ projects for testing.</Description>
    <ResultScale>Seconds</ResultScale>
    <Proportion>LIB</Proportion>
    <TimesToRun>3</TimesToRun>
  </TestInformation>
  <TestProfile>
    <Version>1.0.0</Version>
    <SupportedPlatforms>Linux</SupportedPlatforms>
    <SoftwareType>Application</SoftwareType>
    <TestType>System</TestType>
    <License>Free</License>
    <Status>Experimental</Status>
    <ExternalDependencies>java, python, build-utilities, git, bison, flex</ExternalDependencies>
    <EnvironmentSize>50000</EnvironmentSize>
    <ProjectURL>https://spark.apache.org/</ProjectURL>
    <RepositoryURL>https://github.com/IBM/spark-tpc-ds-performance-test/</RepositoryURL>
    <Maintainer>Michael Larabel</Maintainer>
    <SystemDependencies>byacc, sed, git</SystemDependencies>
  </TestProfile>
  <TestSettings>
    <Option>
      <DisplayName>Scale Factor</DisplayName>
      <Identifier>scale</Identifier>
      <Menu>
        <Entry>
          <Name>1</Name>
          <Value>1</Value>
          <Message>About 1GB of data</Message>
        </Entry>
        <Entry>
          <Name>10</Name>
          <Value>10</Value>
          <Message>About 10GB of data</Message>
        </Entry>
        <Entry>
          <Name>50</Name>
          <Value>50</Value>
          <Message>About 50GB of data</Message>
        </Entry>
        <Entry>
          <Name>100</Name>
          <Value>100</Value>
          <Message>About 100GB of data</Message>
        </Entry>
        <Entry>
          <Name>500</Name>
          <Value>500</Value>
          <Message>About 500GB of data</Message>
        </Entry>
        <Entry>
          <Name>3000</Name>
          <Value>3000</Value>
          <Message>About 3TB of data; many hours of testing</Message>
        </Entry>
        <Entry>
          <Name>10000</Name>
          <Value>10000</Value>
          <Message>About 10TB of data, extremely time consuming!</Message>
        </Entry>
      </Menu>
    </Option>
  </TestSettings>
</PhoronixTestSuite>