TPC Workflow: Clean up CompCluster options

Author: davidrohr

Detectors/TPC/workflow/include/TPCWorkflow/RecoWorkflow.h

@@ -35,15 +35,15 @@ struct CorrectionMapsLoaderGloOpts;
 namespace reco_workflow
 {
 /// define input and output types of the workflow
-enum struct InputType { PassThrough,      // No processing, just pass through available inputs to the writers, defined by the OutputType
-                        Digitizer,        // directly read digits from channel {TPC:DIGITS}
-                        Digits,           // read digits from file
-                        ClustersHardware, // read hardware clusters in raw page format from file
-                        Clusters,         // read native clusters from file
-                        CompClusters,     // read compressed cluster container
-                        CompClustersCTF,  // compressed clusters from CTF, as flat format
-                        CompClustersFlat, // compressed clusters in flat format, used as input for the entropy encoder
-                        EncodedClusters,  // read encoded clusters
+enum struct InputType { PassThrough,               // No processing, just pass through available inputs to the writers, defined by the OutputType
+                        Digitizer,                 // directly read digits from channel {TPC:DIGITS}
+                        Digits,                    // read digits from file
+                        ClustersHardware,          // read hardware clusters in raw page format from file
+                        Clusters,                  // read native clusters from file
+                        CompClustersRoot,          // read compressed cluster in ROOT format
+                        CompClustersFlat,          // compressed clusters from flat format (e.g. from CTF)
+                        CompClustersFlatForEncode, // compressed clusters in flat format, used as input for the entropy encoder, no gpu-reco
+                        EncodedClusters,           // read encoded clusters
                         ZSRaw,
 };
 
@@ -59,7 +59,8 @@ enum struct OutputType { Digits,
                          ClustersHardware,
                          Clusters,
                          Tracks,
-                         CompClusters,
+                         CompClustersRoot,
+                         CompClustersFlat,
                          EncodedClusters,
                          DisableWriter,
                          SendClustersPerSector,

Detectors/TPC/workflow/src/RecoWorkflow.cxx

@@ -80,16 +80,17 @@ const std::unordered_map<std::string, InputType> InputMap{
   {"clustershardware", InputType::ClustersHardware},
   {"clusters", InputType::Clusters},
   {"zsraw", InputType::ZSRaw},
-  {"compressed-clusters", InputType::CompClusters},
-  {"compressed-clusters-ctf", InputType::CompClustersCTF},
-  {"compressed-clusters-flat", InputType::CompClustersFlat}};
+  {"compressed-clusters-root", InputType::CompClustersRoot},
+  {"compressed-clusters-flat", InputType::CompClustersFlat},
+  {"compressed-clusters-flat-for-encode", InputType::CompClustersFlatForEncode}};
 
 const std::unordered_map<std::string, OutputType> OutputMap{
   {"digits", OutputType::Digits},
   {"clustershardware", OutputType::ClustersHardware},
   {"clusters", OutputType::Clusters},
   {"tracks", OutputType::Tracks},
-  {"compressed-clusters", OutputType::CompClusters},
+  {"compressed-clusters-root", OutputType::CompClustersRoot},
+  {"compressed-clusters-flat", OutputType::CompClustersFlat},
   {"encoded-clusters", OutputType::EncodedClusters},
   {"disable-writer", OutputType::DisableWriter},
   {"send-clusters-per-sector", OutputType::SendClustersPerSector},
@@ -122,14 +123,19 @@ framework::WorkflowSpec getWorkflow(CompletionPolicyData* policyData, std::vecto
     throw std::invalid_argument("filtered-input option must be provided only with pass-through input and clusters,tracks,send-clusters-per-sector output");
   }
 
-  bool decompressTPC = inputType == InputType::CompClustersCTF || inputType == InputType::CompClusters;
+  bool decompressTPC = inputType == InputType::CompClustersFlat || inputType == InputType::CompClustersRoot;
   // Disable not applicable settings depending on TPC input, no need to disable manually
   if (decompressTPC && (isEnabled(OutputType::Clusters) || isEnabled(OutputType::Tracks))) {
     caClusterer = false;
     zsOnTheFly = false;
     propagateMC = false;
   }
-  if (inputType == InputType::ZSRaw || inputType == InputType::CompClustersFlat) {
+  if (inputType == InputType::CompClustersFlatForEncode || inputType == InputType::CompClustersRoot || inputType == InputType::CompClustersFlat) {
+    caClusterer = false;
+    zsOnTheFly = false;
+    propagateMC = false;
+  }
+  if (inputType == InputType::ZSRaw) {
     caClusterer = true;
     zsOnTheFly = false;
     propagateMC = false;
@@ -225,7 +231,7 @@ framework::WorkflowSpec getWorkflow(CompletionPolicyData* policyData, std::vecto
       if (sclOpts.requestCTPLumi) { // need CTP digits (lumi) reader
         specs.emplace_back(o2::ctp::getDigitsReaderSpec(false));
       }
-    } else if (inputType == InputType::CompClusters) {
+    } else if (inputType == InputType::CompClustersRoot) {
       // TODO: need to check if we want to store the MC labels alongside with compressed clusters
       // for the moment reading of labels is disabled (last parameter is false)
       // TODO: make a different publisher spec for only one output spec, for now using the
@@ -248,8 +254,9 @@ framework::WorkflowSpec getWorkflow(CompletionPolicyData* policyData, std::vecto
   // output matrix
   // Note: the ClusterHardware format is probably a deprecated legacy format and also the
   // ClusterDecoderRawSpec
-  bool produceCompClusters = isEnabled(OutputType::CompClusters);
-  bool runGPUReco = (produceTracks || produceCompClusters || (isEnabled(OutputType::Clusters) && caClusterer) || inputType == InputType::CompClustersCTF) && inputType != InputType::CompClustersFlat;
+  bool produceCompClustersRoot = isEnabled(OutputType::CompClustersRoot);
+  bool produceCompClustersFlat = isEnabled(OutputType::CompClustersFlat);
+  bool runGPUReco = (produceTracks || produceCompClustersRoot || produceCompClustersFlat || (isEnabled(OutputType::Clusters) && caClusterer) || inputType == InputType::CompClustersFlat) && inputType != InputType::CompClustersFlatForEncode;
   bool runHWDecoder = !caClusterer && (runGPUReco || isEnabled(OutputType::Clusters));
   bool runClusterer = !caClusterer && (runHWDecoder || isEnabled(OutputType::ClustersHardware));
   bool zsDecoder = inputType == InputType::ZSRaw;
@@ -460,13 +467,13 @@ framework::WorkflowSpec getWorkflow(CompletionPolicyData* policyData, std::vecto
     cfg.enableMShape = sclOpts.enableMShapeCorrection;
     cfg.enableCTPLumi = sclOpts.requestCTPLumi;
     cfg.decompressTPC = decompressTPC;
-    cfg.decompressTPCFromROOT = decompressTPC && inputType == InputType::CompClusters;
+    cfg.decompressTPCFromROOT = decompressTPC && inputType == InputType::CompClustersRoot;
     cfg.caClusterer = caClusterer;
     cfg.zsDecoder = zsDecoder;
     cfg.zsOnTheFly = zsOnTheFly;
     cfg.outputTracks = produceTracks;
-    cfg.outputCompClusters = produceCompClusters;
-    cfg.outputCompClustersFlat = runClusterEncoder;
+    cfg.outputCompClustersRoot = produceCompClustersRoot;
+    cfg.outputCompClustersFlat = produceCompClustersFlat || runClusterEncoder;
     cfg.outputCAClusters = isEnabled(OutputType::Clusters) && (caClusterer || decompressTPC);
     cfg.outputQA = isEnabled(OutputType::QA);
     cfg.outputSharedClusterMap = (isEnabled(OutputType::Clusters) || inputType == InputType::Clusters) && isEnabled(OutputType::Tracks) && !isEnabled(OutputType::NoSharedClusterMap);
@@ -500,7 +507,7 @@ framework::WorkflowSpec getWorkflow(CompletionPolicyData* policyData, std::vecto
   //
   // selected by output type 'encoded-clusters'
   if (runClusterEncoder) {
-    specs.emplace_back(o2::tpc::getEntropyEncoderSpec(!runGPUReco && inputType != InputType::CompClustersFlat, selIR));
+    specs.emplace_back(o2::tpc::getEntropyEncoderSpec(!runGPUReco && inputType != InputType::CompClustersFlatForEncode, selIR));
   }
 
   //////////////////////////////////////////////////////////////////////////////////////////////
@@ -547,7 +554,7 @@ framework::WorkflowSpec getWorkflow(CompletionPolicyData* policyData, std::vecto
   // a writer process for compressed clusters container
   //
   // selected by output type 'compressed-clusters'
-  if (produceCompClusters && !isEnabled(OutputType::DisableWriter)) {
+  if (produceCompClustersRoot && !isEnabled(OutputType::DisableWriter)) {
     // defining the track writer process using the generic RootTreeWriter and generator tool
     //
     // defaults

Detectors/TPC/workflow/src/tpc-reco-workflow.cxx

@@ -57,8 +57,8 @@ void customize(std::vector<o2::framework::ConfigParamSpec>& workflowOptions)
   using namespace o2::framework;
 
   std::vector<ConfigParamSpec> options{
-    {"input-type", VariantType::String, "digits", {"digitizer, digits, zsraw, clustershw, clusters, compressed-clusters, compressed-clusters-ctf, pass-through"}},
-    {"output-type", VariantType::String, "tracks", {"digits, zsraw, clustershw, clusters, tracks, compressed-clusters, encoded-clusters, disable-writer, send-clusters-per-sector, qa, no-shared-cluster-map, tpc-triggers"}},
+    {"input-type", VariantType::String, "digits", {"digitizer, digits, zsraw, clustershw, clusters, compressed-clusters-root, compressed-clusters-ctf, compressed-clusters-flat-for-encode, pass-through"}},
+    {"output-type", VariantType::String, "tracks", {"digits, zsraw, clustershw, clusters, tracks, compressed-clusters-root, compressed-clusters-flat, encoded-clusters, disable-writer, send-clusters-per-sector, qa, no-shared-cluster-map, tpc-triggers"}},
     {"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input reader"}},
     {"no-ca-clusterer", VariantType::Bool, false, {"Use HardwareClusterer instead of clusterer of GPUCATracking"}},
     {"disable-mc", VariantType::Bool, false, {"disable sending of MC information"}},

GPU/Workflow/include/GPUWorkflow/GPUWorkflowSpec.h

@@ -117,7 +117,7 @@ class GPURecoWorkflowSpec : public o2::framework::Task
     bool zsDecoder = false;
     bool zsOnTheFly = false;
     bool outputTracks = false;
-    bool outputCompClusters = false;
+    bool outputCompClustersRoot = false;
     bool outputCompClustersFlat = false;
     bool outputCAClusters = false;
     bool outputQA = false;

GPU/Workflow/src/GPUWorkflowSpec.cxx

@@ -236,14 +236,14 @@ void GPURecoWorkflowSpec::init(InitContext& ic)
   }
 
   // Configure the "GPU workflow" i.e. which steps we run on the GPU (or CPU)
-  if (mSpecConfig.outputTracks || mSpecConfig.outputCompClusters || mSpecConfig.outputCompClustersFlat) {
+  if (mSpecConfig.outputTracks || mSpecConfig.outputCompClustersRoot || mSpecConfig.outputCompClustersFlat) {
     mConfig->configWorkflow.steps.set(GPUDataTypes::RecoStep::TPCConversion,
                                       GPUDataTypes::RecoStep::TPCSectorTracking,
                                       GPUDataTypes::RecoStep::TPCMerging);
     mConfig->configWorkflow.outputs.set(GPUDataTypes::InOutType::TPCMergedTracks);
     mConfig->configWorkflow.steps.setBits(GPUDataTypes::RecoStep::TPCdEdx, mConfParam->rundEdx == -1 ? !mConfParam->synchronousProcessing : mConfParam->rundEdx);
   }
-  if (mSpecConfig.outputCompClusters || mSpecConfig.outputCompClustersFlat) {
+  if (mSpecConfig.outputCompClustersRoot || mSpecConfig.outputCompClustersFlat) {
     mConfig->configWorkflow.steps.setBits(GPUDataTypes::RecoStep::TPCCompression, true);
     mConfig->configWorkflow.outputs.setBits(GPUDataTypes::InOutType::TPCCompressedClusters, true);
   }
@@ -966,7 +966,7 @@ void GPURecoWorkflowSpec::run(ProcessingContext& pc)
     LOG(info) << "found " << ptrs.nOutputTracksTPCO2 << " track(s)";
   }
 
-  if (mSpecConfig.outputCompClusters) {
+  if (mSpecConfig.outputCompClustersRoot) {
     o2::tpc::CompressedClustersROOT compressedClusters = *ptrs.tpcCompressedClusters;
     pc.outputs().snapshot(Output{gDataOriginTPC, "COMPCLUSTERS", 0}, ROOTSerialized<o2::tpc::CompressedClustersROOT const>(compressedClusters));
   }
@@ -1259,7 +1259,7 @@ Outputs GPURecoWorkflowSpec::outputs()
   if (mSpecConfig.processMC && mSpecConfig.outputTracks) {
     outputSpecs.emplace_back(gDataOriginTPC, "TRACKSMCLBL", 0, Lifetime::Timeframe);
   }
-  if (mSpecConfig.outputCompClusters) {
+  if (mSpecConfig.outputCompClustersRoot) {
     outputSpecs.emplace_back(gDataOriginTPC, "COMPCLUSTERS", 0, Lifetime::Timeframe);
   }
   if (mSpecConfig.outputCompClustersFlat) {

GPU/Workflow/src/gpu-reco-workflow.cxx

@@ -51,8 +51,8 @@ void customize(std::vector<ConfigParamSpec>& workflowOptions)
 {
 
   std::vector<ConfigParamSpec> options{
-    {"input-type", VariantType::String, "digits", {"digitizer, digits, zsraw, zsonthefly, clustersnative, compressed-clusters-root, compressed-clusters-ctf, trd-tracklets, its-clusters"}},
-    {"output-type", VariantType::String, "tracks", {"clustersnative, tracks, compressed-clusters-ctf, qa, no-shared-cluster-map, send-clusters-per-sector, trd-tracks, error-qa, tpc-triggers, its-tracks"}},
+    {"input-type", VariantType::String, "digits", {"digits, zsraw, zsonthefly, clusters, compressed-clusters-root, compressed-clusters-flat, trd-tracklets, its-clusters, its-mean-vertex"}},
+    {"output-type", VariantType::String, "tracks", {"cluster, tracks, compressed-clusters-root, compressed-clusters-flat, qa, error-qa, no-shared-cluster-map, send-clusters-per-sector, trd-tracks, tpc-triggers, its-tracks"}},
     {"corrmap-lumi-mode", VariantType::Int, 0, {"scaling mode: (default) 0 = static + scale * full; 1 = full + scale * derivative"}},
     {"disable-root-input", VariantType::Bool, true, {"disable root-files input reader"}},
     {"disable-mc", VariantType::Bool, false, {"disable sending of MC information"}},
@@ -98,7 +98,7 @@ enum struct ioType { Digits,
                      ZSRaw,
                      ZSRawOTF,
                      CompClustROOT,
-                     CompClustCTF,
+                     CompClustFlat,
                      Tracks,
                      QA,
                      ErrorQA,
@@ -117,7 +117,7 @@ static const std::unordered_map<std::string, ioType> InputMap{
   {"zsraw", ioType::ZSRaw},
   {"zsonthefly", ioType::ZSRawOTF},
   {"compressed-clusters-root", ioType::CompClustROOT},
-  {"compressed-clusters-ctf", ioType::CompClustCTF},
+  {"compressed-clusters-flat", ioType::CompClustFlat},
   {"trd-tracklets", ioType::TRDTracklets},
   {"its-clusters", ioType::ITSClusters},
   {"its-mean-vertex", ioType::MeanVertex},
@@ -126,7 +126,8 @@ static const std::unordered_map<std::string, ioType> InputMap{
 static const std::unordered_map<std::string, ioType> OutputMap{
   {"clusters", ioType::Clusters},
   {"tracks", ioType::Tracks},
-  {"compressed-clusters-ctf", ioType::CompClustCTF},
+  {"compressed-clusters-flat", ioType::CompClustFlat},
+  {"compressed-clusters-root", ioType::CompClustROOT},
   {"qa", ioType::QA},
   {"error-qa", ioType::ErrorQA},
   {"no-shared-cluster-map", ioType::NoSharedMap},
@@ -167,13 +168,13 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc)
   cfg.enableMShape = sclOpt.enableMShapeCorrection;
   cfg.enableCTPLumi = sclOpt.requestCTPLumi;
   cfg.decompressTPCFromROOT = isEnabled(inputTypes, ioType::CompClustROOT);
-  cfg.decompressTPC = isEnabled(inputTypes, ioType::CompClustCTF) || cfg.decompressTPCFromROOT;
+  cfg.decompressTPC = isEnabled(inputTypes, ioType::CompClustFlat) || cfg.decompressTPCFromROOT;
   cfg.zsDecoder = isEnabled(inputTypes, ioType::ZSRaw);
   cfg.zsOnTheFly = isEnabled(inputTypes, ioType::ZSRawOTF);
   cfg.caClusterer = cfg.zsDecoder || cfg.zsOnTheFly || isEnabled(inputTypes, ioType::Digits);
   cfg.outputTracks = isEnabled(outputTypes, ioType::Tracks);
-  cfg.outputCompClusters = isEnabled(outputTypes, ioType::CompClustROOT);
-  cfg.outputCompClustersFlat = isEnabled(outputTypes, ioType::CompClustCTF);
+  cfg.outputCompClustersRoot = isEnabled(outputTypes, ioType::CompClustROOT);
+  cfg.outputCompClustersFlat = isEnabled(outputTypes, ioType::CompClustFlat);
   cfg.outputCAClusters = isEnabled(outputTypes, ioType::Clusters);
   cfg.outputQA = isEnabled(outputTypes, ioType::QA);
   cfg.outputErrorQA = isEnabled(outputTypes, ioType::ErrorQA);

prodtests/full-system-test/dpl-workflow.sh

@@ -205,7 +205,7 @@ workflow_has_parameter CALIB && [[ $CALIB_TPC_VDRIFTTGL == 1 ]] && SEND_ITSTPC_D
 
 PVERTEXING_CONFIG_KEY+="${ITSMFT_STROBES};"
 
-has_processing_step ENTROPY_ENCODER && has_detector_ctf TPC && GPU_OUTPUT+=",compressed-clusters-ctf"
+has_processing_step ENTROPY_ENCODER && has_detector_ctf TPC && GPU_OUTPUT+=",compressed-clusters-flat"
 
 if [[ $SYNCMODE == 1 ]] && workflow_has_parameter QC && has_detector_qc TPC; then
   GPU_OUTPUT+=",qa,error-qa"
@@ -443,7 +443,7 @@ fi
 
 if [[ -n $INPUT_DETECTOR_LIST ]]; then
   if [[ $CTFINPUT == 1 ]]; then
-    GPU_INPUT=compressed-clusters-ctf
+    GPU_INPUT=compressed-clusters-flat
     TOF_INPUT=digits
     CTFName=`ls -t $RAWINPUTDIR/o2_ctf_*.root 2> /dev/null | head -n1`
     [[ -z $CTFName && $WORKFLOWMODE == "print" ]] && CTFName='$CTFName'
@@ -652,7 +652,7 @@ if has_processing_step ENTROPY_ENCODER && [[ -n "$WORKFLOW_DETECTORS_CTF" ]] &&
   has_detector_ctf TOF && add_W o2-tof-entropy-encoder-workflow "$RANS_OPT --mem-factor ${TOF_ENC_MEMFACT:-1.5} --pipeline $(get_N tof-entropy-encoder TOF CTF 1)"
   has_detector_ctf ITS && add_W o2-itsmft-entropy-encoder-workflow "$RANS_OPT --mem-factor ${ITS_ENC_MEMFACT:-1.5} --pipeline $(get_N its-entropy-encoder ITS CTF 1)"
   has_detector_ctf TRD && add_W o2-trd-entropy-encoder-workflow "$RANS_OPT --mem-factor ${TRD_ENC_MEMFACT:-1.5} --pipeline $(get_N trd-entropy-encoder TRD CTF 1 TRDENT)"
-  has_detector_ctf TPC && add_W o2-tpc-reco-workflow " $RANS_OPT --mem-factor ${TPC_ENC_MEMFACT:-1.} --input-type compressed-clusters-flat --output-type encoded-clusters,disable-writer --pipeline $(get_N tpc-entropy-encoder TPC CTF 1 TPCENT)"
+  has_detector_ctf TPC && add_W o2-tpc-reco-workflow " $RANS_OPT --mem-factor ${TPC_ENC_MEMFACT:-1.} --input-type compressed-clusters-flat-for-encode --output-type encoded-clusters,disable-writer --pipeline $(get_N tpc-entropy-encoder TPC CTF 1 TPCENT)"
   has_detector_ctf CTP && add_W o2-ctp-entropy-encoder-workflow "$RANS_OPT --mem-factor ${CTP_ENC_MEMFACT:-1.5} --pipeline $(get_N its-entropy-encoder CTP CTF 1)"
 
   if [[ $CREATECTFDICT == 1 && $WORKFLOWMODE == "run" ]] ; then