[WIP] Get CCDB time machine timestamp from CCDB by default in anchorMC.sh

GRID/utils/getCCDBTimeMachineTimestamp.py

@@ -211,7 +211,7 @@ def query_CCDBTimeMachineTS(base_url, user, period, passname):
     Queries CCDBTimeMachineTS from CCDB for a given period and pass.
 
     Returns:
-        obtained NotAfter timestamp (int) or None
+        (tuple) (obtained NotAfter timestamp (int), True) or (None, False)
     """
     # Construct user ID-specific cert and key paths
     cert_path, key_path = make_cert_paths()
@@ -235,9 +235,9 @@ def filter_known_fields(cls, data: dict) -> dict:
           print(f"[getCCDBTimeMachineTimestamp] Found object for {clean_meta['Period']}, Pass: {clean_meta['Pass']}, "
                 f"NotAfter: {clean_meta['NotAfter']} ({human(clean_meta['NotAfter'])}), uploaded by LPMProductionTag {clean_meta['LPMProductionTag']}",
                 file=sys.stderr)
-          return clean_meta['NotAfter']
+          return clean_meta['NotAfter'], True
 
-    return None
+    return None, False
 
 
 def upload_CCDBTimeMachineTS(base_url, user, keys):
@@ -288,20 +288,27 @@ def get_CCDBTimeMachineTS(production_tag, year, period, run, passname,
     Returns:
         int: Either the queried or the estimated NotAfter timestamp
     """
+    found = False
     if not skip_ccdb_query:
-        not_after = query_CCDBTimeMachineTS(ccdb_url, username, period, passname)
-    else:
-        not_after = None
+        not_after, found = query_CCDBTimeMachineTS(ccdb_url, username, period, passname)
 
-    if not_after is not None and force_overwrite == True:
-        print("[getCCDBTimeMachineTimestamp] Existing object will be overwritten", file=sys.stderr)
-
-    if not_after == None or force_overwrite == True:
+    if not found or force_overwrite:
         print("[getCCDBTimeMachineTimestamp] Estimating CCDBTimeMachineTS for "
               f"LPMProductionTag: {production_tag}, Period: {period}, Pass: {passname}", file=sys.stderr)
         not_after = get_NotAfter(year, period, run, passname, samples, ccdb_check_path, ccdb_check)
-        time_machine_ts = CCDBTimeMachineTS(production_tag, period, passname, not_after)
+
         if publish_to_ccdb:
+            if not found:
+                # check again in case another subjob uploaded an object in the meantime
+                not_after_q, found = query_CCDBTimeMachineTS(ccdb_url, username, period, passname)
+
+            if found:
+                if force_overwrite:
+                    print("[getCCDBTimeMachineTimestamp] Existing object will be overwritten", file=sys.stderr)
+                else:
+                    return not_after_q
+
+            time_machine_ts = CCDBTimeMachineTS(production_tag, period, passname, not_after)
             upload_CCDBTimeMachineTS(ccdb_url, username, dataclasses.asdict(time_machine_ts))
 
     return not_after
@@ -335,11 +342,11 @@ def main():
                         help="overwrite the current CCDBTimeMachineTS object in the CCDB if one exists")
     parser.add_argument("--ccdb-url", default="https://alice-ccdb.cern.ch",
                         help="ccdb host for the upload")
-    parser.add_argument("--username", default="aliprod",
+    parser.add_argument("--username", default=os.environ.get("JALIEN_USER"),
                         help="username to be used for the upload to CCDB")
     args = parser.parse_args()
 
-    missing = [n for n in ("year", "period", "run", "passname") if not getattr(args, n)]
+    missing = [n for n in ("production_tag", "year", "period", "run", "passname", "username") if not getattr(args, n)]
     if missing:
         parser.error("missing required value(s): " + ", ".join(missing))
 

MC/run/ANCHOR/anchorMC.sh

@@ -354,7 +354,29 @@ remainingargs="${ALIEN_JDL_ANCHOR_SIM_OPTIONS} ${remainingargs} --anchor-config
 ALIEN_JDL_O2DPG_ASYNC_RECO_FROMSTAGE=${ALIEN_JDL_O2DPG_ASYNC_RECO_FROMSTAGE:-RECO}
 remainingargs="${remainingargs} ${ALIEN_JDL_O2DPG_ASYNC_RECO_TAG:+--alternative-reco-software ${PWD}/env_async.env@${ALIEN_JDL_O2DPG_ASYNC_RECO_FROMSTAGE}}"
 # potentially add CCDB timemachine timestamp
-remainingargs="${remainingargs} ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER:+--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}}"
+if [[ -n ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER} ]]; then
+  # override CCDB time machine timestamp by hand, use e.g. 3385078236000 to effectively disable it
+  ARG_CONDITION_NOT_AFTER="--condition-not-after ${ALIEN_JDL_CCDB_CONDITION_NOT_AFTER}"
+else
+  # options to set CCDB url, disable upload and force overwrite of an existing timestamp; default to production CCDB, uploading the object and not overwriting
+  : ${CCDBTIMEMACHINE_CCDB_URL:=https://alice-ccdb.cern.ch}
+  [[ ${CCDBTIMEMACHINE_SKIP_UPLOAD:-} != "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --publish-to-ccdb"
+  [[ ${CCDBTIMEMACHINE_FORCE_OVERWRITE:-} == "1" ]] && CCDBTIMEMACHINE_OPTIONS+=" --force-overwrite"
+
+  # obtain CCDB time machine timestamp from CCDB or, if not available, estimate it from the creation time of the reconstruction output files
+  CCDBTIMEMACHINE_CMD="${O2DPG_ROOT}/GRID/utils/getCCDBTimeMachineTimestamp.py --ccdb-url $CCDBTIMEMACHINE_CCDB_URL ${CCDBTIMEMACHINE_OPTIONS:-}"
+  echo "Running command to obtain CCDB time machine timestamp: ${CCDBTIMEMACHINE_CMD}"
+  CCDBTIMEMACHINE_TS=$(${CCDBTIMEMACHINE_CMD})
+
+  # return an error if the timestamp could not be obtained
+  if [[ -n ${CCDBTIMEMACHINE_TS} ]]; then
+    ARG_CONDITION_NOT_AFTER="--condition-not-after ${CCDBTIMEMACHINE_TS}"
+  else
+    echo_error "Could not obtain CCDB time machine timestamp. Exiting."
+    exit 1
+  fi
+fi
+remainingargs="${remainingargs} ${ARG_CONDITION_NOT_AFTER:-}"
 
 echo_info "baseargs passed to o2dpg_sim_workflow_anchored.py: ${baseargs}"
 echo_info "remainingargs forwarded to o2dpg_sim_workflow.py: ${remainingargs}"