1 #!/bin/bash 2 3 # These should be on the local filesystem. We'll be hitting it hard. 4 DATA_DIR=/usr/local/google/home/${USER}/ 5 SYMBOL_CACHE=${DATA_DIR}cache/ 6 REPORT_DIR=${DATA_DIR}reports/ 7 SAMPLE_DIR=${DATA_DIR}samples/ 8 RECORD_FILE=/tmp/profiles.rio 9 COLUMN_FILE=/tmp/profiles.cio 10 mkdir -p ${SYMBOL_CACHE} 11 mkdir -p ${REPORT_DIR} 12 mkdir -p ${SAMPLE_DIR} 13 14 # Directory that has the scripts app_engine_pull.py and symbolizer.py 15 INTERPRETER_DIR=/google/src/files/p2/head/depot2/gcctools/chromeos/v14/cwp/interpreter/ 16 V14_DIR=$(dirname $(dirname ${INTERPRETER_DIR})) 17 18 PYTHONPATH=$PYTHONPATH:$V14_DIR 19 20 # Profile util binary 21 PROFILE_UTIL_BINARY=/home/mrdmnd/${USER}-profiledb/google3/blaze-bin/perftools/gwp/chromeos/profile_util 22 23 # mr-convert binary 24 MR_CONVERT_BINARY=/home/build/static/projects/dremel/mr-convert 25 26 CNS_LOC=/cns/ag-d/home/${USER}/profiledb/ 27 28 # Protofile location 29 PROTO_LOC=${CNS_LOC}cwp_profile_db_entry.proto 30 31 echo "0. Cleaning up old data." 32 rm /tmp/profiles.* 33 rm ${REPORT_DIR}* 34 rm ${SAMPLE_DIR}* 35 36 37 echo "Starting CWP Pipeline..." 38 echo "1. Pulling samples to local filesystem from server." 39 echo " For demo purposes, UN=${USER}@google.com, PW=xjpbmshkzefutlrm" 40 python ${INTERPRETER_DIR}app_engine_pull.py --output_dir=${SAMPLE_DIR} 41 echo "2. Symbolizing samples to perf reports. Hold on..." 42 43 python ${INTERPRETER_DIR}symbolizer.py --in=${SAMPLE_DIR} --out=${REPORT_DIR} --cache=${SYMBOL_CACHE} 44 echo "3. Loading reports into RecordIO format..." 45 # Will need to make append_dir more clever / incremental 46 ${PROFILE_UTIL_BINARY} --record=${RECORD_FILE} --append_dir=${REPORT_DIR} 47 echo "Done." 48 echo "4. Converting records to columnio." 49 ${MR_CONVERT_BINARY} --mapreduce_input_map=recordio:${RECORD_FILE} --mapreduce_output_map=${COLUMN_FILE}@1 --columnio_mroutput_message_type=CwpProfileDbEntry --columnio_mroutput_protofiles=${PROTO_LOC} 50 echo "5. Uploading columnio to colossus." 51 fileutil cp -f /tmp/profiles.cio-* ${CNS_LOC} 52 echo "6. Let's try some dremel queries..." 53 echo " dremel> define table t /cns/ag-d/home/${USER}/profiledb/profiles.cio-*" 54 echo " Like, say, dremel> select sum(frames.count) as count, left(frames.function_name, 80) as name from t group by name order by count desc limit 25;" 55 56
