CCExtractor · canihavesomecoffee · Aug 1, 2026 · Jul 20, 2026
@@ -0,0 +1,37 @@
+name: CI-VM scripts
+
+# Static analysis for the CI-VM runner scripts (the scripts that run on the
+# throwaway test VMs). Runs only when those scripts change. shellcheck lints the
+# bash runners; PSScriptAnalyzer lints the PowerShell startup script.
+
+on:
+  pull_request:
+    paths:
+      - 'install/ci-vm/**'
+      - '.github/workflows/ci-vm-scripts.yml'
+  push:
+    paths:
+      - 'install/ci-vm/**'
+      - '.github/workflows/ci-vm-scripts.yml'
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install shellcheck
+        run: sudo apt-get update && sudo apt-get install -y shellcheck
+      - name: shellcheck (bash runner scripts)
+        run: shellcheck --severity=error install/ci-vm/ci-linux/ci/runCI install/ci-vm/ci-linux/startup-script.sh
+
+  psscriptanalyzer:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: PSScriptAnalyzer (startup-script.ps1)
+        shell: pwsh
+        run: |
+          Set-PSRepository PSGallery -InstallationPolicy Trusted
+          Install-Module -Name PSScriptAnalyzer -Force -Scope CurrentUser
+          $issues = Invoke-ScriptAnalyzer -Path install/ci-vm/ci-windows/startup-script.ps1 -Severity Error
+          if ($issues) { $issues | Format-Table -AutoSize; exit 1 }
@@ -103,6 +103,27 @@ function executeCommand {
         fi
 }
 
+# Send a build artifact to the server (best-effort: a failed upload must never
+# abort the run, so this always returns 0). Reuses the reportURL/curl pattern.
+function sendArtifact {
+    local artifactType="$1"
+    local filePath="$2"
+    if [ ! -f "${filePath}" ]; then
+        echo "Artifact ${artifactType}: no file at '${filePath}', skipping" >> "${logFile}"
+        return 0
+    fi
+    echo "Uploading ${artifactType} artifact (${filePath})" >> "${logFile}"
+    # --fail surfaces HTTP >= 400 (e.g. 413 on an oversized coredump) as a
+    # curl error so failed uploads are visible in the log; still best-effort.
+    curl -s --fail -A "${userAgent}" --form "type=artifactupload" --form "artifact_type=${artifactType}" \
+        --form "file=@${filePath}" "${reportURL}" >> "${logFile}" 2>&1
+    local curlStatus=$?
+    if [ ${curlStatus} -ne 0 ]; then
+        echo "Artifact ${artifactType}: upload failed (curl exit ${curlStatus})" >> "${logFile}"
+    fi
+    return 0
+}
+
 # Source variables
 . "$DIR/variables"
 
@@ -125,7 +146,25 @@ if [ -e "${dstDir}/ccextractor" ]; then
         echo "=== End Version Info ===" >> "${logFile}"
         postStatus "testing" "Running tests"
         executeCommand cd ${suiteDstDir}
-        executeCommand ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}"
+
+        # Enable core dumps and capture the test run's combined stdout/stderr so
+        # both can be uploaded as artifacts (the API serves them per run).
+        ulimit -c unlimited 2>/dev/null
+        echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null 2>&1
+        combinedLog="${reportFolder}/combined_stdout.log"
+        ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" > "${combinedLog}" 2>&1
+        testerStatus=$?
+        cat "${combinedLog}" >> "${logFile}"
+
+        # Upload artifacts before any failure-halt so crash data survives.
+        sendArtifact "binary" "${dstDir}/ccextractor"
+        sendArtifact "combined_stdout" "${combinedLog}"
+        sendArtifact "coredump" "$(ls -1t core.* 2>/dev/null | head -n1)"
+
+        if [ ${testerStatus} -ne 0 ]; then
+                haltAndCatchFire ""
+        fi
+
         sendLogFile
         postStatus "completed" "Ran all tests"
 

@@ -4,7 +4,7 @@ curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v3.2
 dpkg --install gcsfuse_3.2.0_amd64.deb
 rm gcsfuse_3.2.0_amd64.deb
 
-apt install gnupg ca-certificates
+apt install -y gnupg ca-certificates
 gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
 echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
 sudo apt update
@@ -14,7 +14,8 @@ mkdir repository
 cd repository
 
 # Use gcsfuse and import required files
-mkdir temp TestFiles TestResults vm_data reports
+# TempFiles is used by the tester (--tempfolder) and must exist
+mkdir temp TestFiles TestResults TempFiles vm_data reports
 
 gcs_bucket=$(curl http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google")
 
@@ -31,6 +32,9 @@ mount vm_data
 mount TestFiles
 mount TestResults
 
+# Give gcsfuse mounts time to become ready
+sleep 10
+
 cp temp/* ./
 
 chmod +x bootstrap

@@ -15,7 +15,10 @@ for /F %%R in ('curl http://metadata/computeMetadata/v1/instance/attributes/repo
 SET userAgent="CCX/CI_BOT"
 SET logFile="%reportFolder%/log.html"
 
-call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact" >> "%logFile%"
+rem NB: no outer ">> %logFile%" here. postStatus already appends to %logFile%
+rem internally; an outer redirect to the same file self-locks on Windows
+rem (the inner append cannot open a file the outer redirect holds open).
+call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact"
 
 echo Checking for CCExtractor build artifact
 if EXIST "%dstDir%\ccextractorwinfull.exe" (
@@ -27,7 +30,21 @@ if EXIST "%dstDir%\ccextractorwinfull.exe" (
     echo === End Version Info === >> "%logFile%"
     call :postStatus "testing" "Running tests"
     call :executeCommand cd %suiteDstDir%
-    call :executeCommand "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%"
+
+    rem Capture the test run's combined stdout/stderr for upload as an artifact.
+    rem NB: "||" instead of "if errorlevel 1" - it fires at runtime on ANY
+    rem nonzero exit code, including the negative NTSTATUS codes a crash
+    rem returns, e.g. 0xC0000005, and works inside this parenthesized block
+    rem where %ERRORLEVEL% would expand too early.
+    set "testerFailed="
+    "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%" > "%reportFolder%/combined_stdout.log" 2>&1 || set "testerFailed=1"
+    type "%reportFolder%/combined_stdout.log" >> "%logFile%"
+
+    rem Upload artifacts (best-effort; never aborts the run). Windows skips coredump for v1.
+    call :sendArtifact "binary" "%dstDir%\ccextractorwinfull.exe"
+    call :sendArtifact "combined_stdout" "%reportFolder%/combined_stdout.log"
+
+    if defined testerFailed call :haltAndCatchFire ""
 
     call :sendLogFile
 
@@ -144,3 +161,21 @@ if !sl_attempt! LEQ %sl_max_retries% (
 echo ERROR: Failed to upload log after %sl_max_retries% attempts >> "%logFile%"
 endlocal
 EXIT /B 1
+
+rem Send a build artifact to the server (best-effort; never aborts the run)
+:sendArtifact
+setlocal
+set "sa_type=%~1"
+set "sa_file=%~2"
+if NOT EXIST "%sa_file%" (
+    echo Artifact %sa_type%: no file at "%sa_file%", skipping >> "%logFile%"
+    endlocal
+    EXIT /B 0
+)
+echo Uploading %sa_type% artifact (%sa_file%) >> "%logFile%"
+rem --fail makes HTTP >= 400 (e.g. 413 on an oversized file) a curl error so
+rem the failure is at least visible in the log; the upload stays best-effort.
+curl -s --fail -A "%userAgent%" --form "type=artifactupload" --form "artifact_type=%sa_type%" --form "file=@%sa_file%" "%reportURL%" >> "%logFile%" 2>&1
+if errorlevel 1 echo Artifact %sa_type%: upload failed with curl exit code %ERRORLEVEL% >> "%logFile%"
+endlocal
+EXIT /B 0
@@ -1,25 +1,34 @@
 """
-System, health, and queue routes.
+System, health, queue, and artifact routes.
 
 GET /system/health           Health check (unauthenticated)
 GET /system/queue            Queue status — active + queued runs
+GET /runs/{id}/artifacts     Run artifacts from GCS + local storage
 """
 
 import os
 from datetime import datetime, timezone
 
 from flask import g, jsonify, request
 from sqlalchemy import text
+from sqlalchemy.orm import joinedload
 
 from mod_api import mod_api
 from mod_api.middleware.auth import require_scope
 from mod_api.middleware.error_handler import make_error_response
-from mod_api.middleware.validation import validate_offset_pagination
+from mod_api.middleware.validation import (validate_offset_pagination,
+                                           validate_path_id)
 from mod_api.models.api_token import Scope
 from mod_api.schemas.common import DATETIME_FORMAT
-from mod_api.services.status import batch_get_run_data
-from mod_api.utils import paginated_response
-from mod_test.models import Test, TestPlatform, TestProgress, TestStatus
+from mod_api.services.status import batch_get_run_data, is_dummy_row
+from mod_api.services.storage import (get_log_file_path,
+                                      get_test_results_base_path,
+                                      resolve_artifact)
+from mod_api.utils import paginated_response, safe_resolve
+from mod_test.models import (Test, TestPlatform, TestProgress, TestResultFile,
+                             TestStatus)
+
+OCTET_STREAM = 'application/octet-stream'
 
 
 @mod_api.route('/system/health', methods=['GET'])
@@ -202,3 +211,137 @@ def get_queue(limit=50, offset=0):
             'running_count': running_count,
         }
     )
+
+
+def _get_gcs_artifacts(run_id, platform):
+    binary_name = (
+        'ccextractor' if platform == TestPlatform.linux
+        else 'ccextractorwinfull.exe'
+    )
+    gcs_artifacts = [
+        ('binary',
+         f'test_artifacts/{run_id}/{binary_name}', binary_name, OCTET_STREAM),
+        ('coredump', f'test_artifacts/{run_id}/coredump',
+         f'coredump-{run_id}', OCTET_STREAM),
+        (
+            'combined_stdout',
+            f'test_artifacts/{run_id}/combined_stdout.log',
+            f'combined_stdout-{run_id}.log',
+            'text/plain',
+        ),
+    ]
+    artifacts = []
+    for artifact_type, gcs_path, filename, content_type in gcs_artifacts:
+        download_url, storage_status = resolve_artifact(gcs_path)
+        artifacts.append({
+            'artifact_id': f'{artifact_type}_{run_id}',
+            'run_id': run_id,
+            'sample_id': None,
+            'type': artifact_type,
+            'filename': filename,
+            'content_type': content_type,
+            'size_bytes': None,
+            'storage_status': storage_status,
+            'download_url': download_url,
+        })
+    return artifacts
+
+
+def _get_output_artifacts(run_id):
+    result_files = TestResultFile.query.options(
+        joinedload(TestResultFile.regression_test_output),
+        joinedload(TestResultFile.regression_test),
+    ).filter_by(test_id=run_id).all()
+    for rf in result_files:
+        if is_dummy_row(rf):
+            continue
+
+        ext = rf.regression_test_output.correct_extension if rf.regression_test_output else ''
+        sample_id = (rf.regression_test.sample_id
+                     if rf.regression_test else None)
+
+        expected_name = rf.expected + ext
+        # NOTE: storage metadata (storage_status, download_url, size_bytes,
+        # content_type) is resolved by list_artifacts for paged items only.
+
+        yield {
+            'artifact_id': f'expected_{run_id}_{rf.regression_test_id}_{rf.regression_test_output_id}',
+            'run_id': run_id,
+            'sample_id': sample_id,
+            'type': 'expected_output',
+            'filename': expected_name,
+        }
+
+        if rf.got is not None:
+            actual_name = rf.got + ext
+            yield {
+                'artifact_id': f'actual_{run_id}_{rf.regression_test_id}_{rf.regression_test_output_id}',
+                'run_id': run_id,
+                'sample_id': sample_id,
+                'type': 'actual_output',
+                'filename': actual_name,
+            }
+
+
+@mod_api.route('/runs/<run_id>/artifacts', methods=['GET'])
+@require_scope(Scope.RESULTS_READ)
+@validate_path_id('run_id')
+@validate_offset_pagination()
+def list_artifacts(run_id, limit=50, offset=0):
+    """
+    List all artifacts for a run.
+
+    Checks both GCS and local storage. Falls back to local when GCS
+    is unavailable. Supports ?type filter.
+    """
+    test = Test.query.filter(Test.id == run_id).first()
+    if test is None:
+        return make_error_response(
+            'not_found',
+            f'Run {run_id} not found.',
+            http_status=404)
+
+    artifacts = _get_gcs_artifacts(run_id, test.platform)
+
+    # Build log — accessed via /runs/{id}/logs, no direct download link.
+    log_path = get_log_file_path(run_id)
+    artifacts.append({
+        'artifact_id': f'buildlog_{run_id}',
+        'run_id': run_id,
+        'sample_id': None,
+        'type': 'build_log',
+        'filename': f'{run_id}.txt',
+        'content_type': 'text/plain',
+        'size_bytes': os.path.getsize(log_path) if log_path else None,
+        'storage_status': 'ok' if log_path else 'missing',
+        'download_url': None,
+    })
+
+    artifacts.extend(list(_get_output_artifacts(run_id)))
+
+    # Apply optional ?type filter.
+    type_filter = request.args.get('type')
+    if type_filter:
+        artifacts = [a for a in artifacts if a['type'] == type_filter]
+
+    total = len(artifacts)
+    paged = artifacts[offset:offset + limit]
+
+    # Resolve heavy artifact metadata only for the returned page
+    base_path = get_test_results_base_path()
+    for a in paged:
+        if 'storage_status' not in a:
+            # It's an output artifact
+            filename = a['filename']
+            url, status = resolve_artifact(f'TestResults/{filename}')
+            local = safe_resolve(base_path, filename)
+
+            a['content_type'] = OCTET_STREAM
+            a['size_bytes'] = (
+                os.path.getsize(local)
+                if local and os.path.isfile(local) else None
+            )
+            a['storage_status'] = status
+            a['download_url'] = url
+
+    return paginated_response(paged, total, limit, offset)