Keep reports off asap-remote, survive mail errors (#416)

.CI/Jenkinsfile

@@ -498,7 +498,7 @@ pipeline {
     stage('report') {
       agent {
         dockerfile {
-          label 'linux'
+          label 'linux && !asap-remote'
           dir '.CI/build-dep'
           customWorkspace 'ws/OpenModelicaLibraryTestingReport'
           args '-e IDA_EMAIL_USR=$IDA_EMAIL_USR -e IDA_EMAIL_PSW=$IDA_EMAIL_PSW'
@@ -546,7 +546,17 @@ pipeline {
         // knows about is never generated again, so one that a later failure in
         // this stage stops from being uploaded would be lost for good.
         sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])])
-        sh './publish-reports.py'
+        // Exit code 2: the reports are recorded but the mail server could not be
+        // reached. The log lists the emails left unsent; the overviews below are
+        // still built, so the build is only marked unstable.
+        script {
+          def status = sh(script: './publish-reports.py', returnStatus: true)
+          if (status == 2) {
+            unstable('publish-reports.py could not send the regression emails, see the log for the ones left unsent')
+          } else if (status != 0) {
+            error("publish-reports.py failed with exit code ${status}")
+          }
+        }
 
         sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json"
         sh 'mv overview.html overview-combined.html'
@@ -628,7 +638,7 @@ pipeline {
     stage('drop stale pull request tables') {
       agent {
         dockerfile {
-          label 'linux'
+          label 'linux && !asap-remote'
           dir '.CI/build-dep'
           customWorkspace 'ws/OpenModelicaLibraryTestingReport'
         }
@@ -652,7 +662,7 @@ pipeline {
     stage('pull request report') {
       agent {
         dockerfile {
-          label 'linux'
+          label 'linux && !asap-remote'
           dir '.CI/build-dep'
           customWorkspace 'ws/OpenModelicaLibraryTestingReport'
         }

publish-reports.py

@@ -30,6 +30,7 @@
   pending = json.load(fin)
 
 entries = pending.get("entries", {})
+unsent = []
 emails = pending.get("emails", {})
 
 db = resultsdb.connect(args.db)
@@ -51,7 +52,8 @@
     missing_html = ("%s %s %s" % ("<p style=\"color:red;\">", missing_plain, "</p"))
 
   sent = 0
-  for email in sorted(emails.keys()):
+  recipients = sorted(emails.keys())
+  for i, email in enumerate(recipients):
     msg = EmailMessage()
     msg['Subject'] = 'OpenModelica Library Testing Regressions'
     msg['From'] = Address("OM Hudson", "openmodelicabuilds", "ida.liu.se")
@@ -70,17 +72,35 @@
 </body>
 </html>
 """ % (missing_html, "\n".join(reversed(emails[email]["html"]))), subtype='html')
-    with smtplib.SMTP('smtp.office365.com') as s:
-      s.starttls()
-      s.ehlo()
-      s.login(os.environ["IDA_EMAIL_USR"],os.environ["IDA_EMAIL_PSW"])
-      try:
-        s.send_message(msg)
-      except smtplib.SMTPRecipientsRefused as e:
-        print("Skipping refused address %s: %s" % (email, e.recipients))
-        continue
+    try:
+      with smtplib.SMTP('smtp.office365.com', timeout=60) as s:
+        s.starttls()
+        s.ehlo()
+        s.login(os.environ["IDA_EMAIL_USR"],os.environ["IDA_EMAIL_PSW"])
+        try:
+          s.send_message(msg)
+        except smtplib.SMTPRecipientsRefused as e:
+          print("Skipping refused address %s: %s" % (email, e.recipients))
+          continue
+    except (OSError, smtplib.SMTPException) as e:
+      # The rows are in the [history] table already, so a rerun would not mail
+      # these regressions again: the log is the only place they are left in.
+      # The mail server is not reachable for the rest of them either, and the
+      # overviews this stage builds next must not be lost over it.
+      unsent = recipients[i:]
+      print("Could not send the emails, %s: %s" % (type(e).__name__, e))
+      print("Not sent, %d of %d emails:" % (len(unsent), len(emails)))
+      for address in unsent:
+        print("  %s:" % address)
+        for line in reversed(emails[address]["plain"]):
+          print("    %s" % line)
+      break
     sent += 1
   print("Sent %d of %d emails" % (sent, len(emails)))
 
 if not args.keep:
   os.remove(args.pending)
+
+if unsent:
+  # Tells the Jenkinsfile to mark the build unstable and go on.
+  sys.exit(2)