From 33330f5a896a05f469789ecfc07eabdf8cc5c575 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Sat, 6 Dec 2025 18:50:13 +0000 Subject: [PATCH 1/2] Added database testing support --- .github/workflows/production-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml index 82ae67c..2adf001 100644 --- a/.github/workflows/production-deploy.yml +++ b/.github/workflows/production-deploy.yml @@ -37,4 +37,6 @@ jobs: DB_PASSWORD: ${{ secrets.DB_PASSWORD }} DB_HOST: ${{ secrets.DB_HOST }} DB_PORT: ${{ secrets.DB_PORT }} - DB_NAME: ${{ secrets.DB_NAME }} \ No newline at end of file + DB_NAME: ${{ secrets.DB_NAME }} + GCP_DB_CREDS: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} \ No newline at end of file From 811a84ac89fb075765a836d3407350210aa72c1f Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Thu, 7 May 2026 10:21:04 +0100 Subject: [PATCH 2/2] Updated workflows --- .github/workflows/pre_production_tests.yml | 49 ++++++++++++++++++ .github/workflows/production-deploy.yml | 30 +++++++++-- .github/workflows/staging-deploy.yml | 59 ++++++---------------- .github/workflows/test-lint.yml | 49 ++++++++++++++++++ 4 files changed, 139 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/pre_production_tests.yml create mode 100644 .github/workflows/test-lint.yml diff --git a/.github/workflows/pre_production_tests.yml b/.github/workflows/pre_production_tests.yml new file mode 100644 index 0000000..2daabd1 --- /dev/null +++ b/.github/workflows/pre_production_tests.yml @@ -0,0 +1,49 @@ +name: Run Pre-Production Validation Tests + +on: + workflow_dispatch: + inputs: + eval_function: + type: string + description: "The name of the evaluation function to test" + required: true + sql_limit: + type: number + description: "The maximum number of SQL test cases to run" + required: false + default: 500 + seed: + type: string + description: "Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate." + required: false + default: '' + request_delay: + type: string + description: "Delay (seconds) between dispatching requests" + required: false + default: '0' + max_concurrency: + type: string + description: "Max concurrent requests (lower for GPT-backed functions)" + required: false + default: '5' + +jobs: + run-pre-production-tests: + name: 🧪 Run Staging Validation Tests + uses: lambda-feedback/Database-Testing/.github/workflows/test_evaluation_function.yml@main + with: + eval_function: ${{ inputs.eval_function }} + sql_limit: ${{ inputs.sql_limit }} + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} + secrets: + TEST_API_ENDPOINT: ${{ secrets.TEST_API_ENDPOINT }} + DB_USER: ${{ secrets.DB_USER }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} + DB_HOST: ${{ secrets.DB_HOST }} + DB_PORT: ${{ secrets.DB_PORT }} + DB_NAME: ${{ secrets.DB_NAME }} + GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_DB_CREDS }} + GCP_PROJECT_ID: ${{ secrets.GCP_DB_PROJECT_ID }} \ No newline at end of file diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml index 2adf001..6229778 100644 --- a/.github/workflows/production-deploy.yml +++ b/.github/workflows/production-deploy.yml @@ -1,4 +1,4 @@ -name: Deploy Production Version to Lambda Feedback +name: Deploy to Production on: workflow_dispatch: @@ -17,15 +17,37 @@ on: required: true type: string default: 'main' + seed: + description: 'Random seed for reproducible sampling (float in [-1.0, 1.0]). Leave blank to auto-generate.' + required: false + type: string + default: '' + request_delay: + description: 'Delay (seconds) between dispatching requests' + required: false + type: string + default: '0' + max_concurrency: + description: 'Max concurrent requests (lower for GPT-backed functions)' + required: false + type: string + default: '5' jobs: deploy: - uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@deploy-request + permissions: + contents: write + packages: write + id-token: write + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main with: - template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-wolfram' + template-repository-name: 'lambda-feedback/evaluation-function-boilerplate-python' environment: "production" version-bump: ${{ inputs.version-bump }} branch: ${{ inputs.branch }} - run-tests: true + run-database-tests: false + seed: ${{ inputs.seed }} + request_delay: ${{ inputs.request_delay }} + max_concurrency: ${{ inputs.max_concurrency }} secrets: aws-key-id: ${{ secrets.LAMBDA_CONTAINER_PIPELINE_AWS_ID }} diff --git a/.github/workflows/staging-deploy.yml b/.github/workflows/staging-deploy.yml index 626819a..24c4d99 100644 --- a/.github/workflows/staging-deploy.yml +++ b/.github/workflows/staging-deploy.yml @@ -1,10 +1,9 @@ -name: Deploy to Lambda Feedback +name: Deploy to Staging on: push: branches: - main - - master workflow_dispatch: jobs: @@ -23,69 +22,41 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - lfs: true - - name: pull lfs files - run: git lfs pull - - name: Verify LFS files - run: git lfs ls-files - name: Set up Python ${{ matrix.python-version }} id: python-setup uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Load cached Poetry installation - id: poetry-cache - uses: actions/cache@v4 - with: - path: ~/.local - key: poetry-0 - - - name: Install and configure Poetry - if: steps.poetry-cache.outputs.cache-hit != 'true' - uses: snok/install-poetry@v1 - with: - virtualenvs-in-project: true - - - name: Load cached venv - id: dependencies-cache - uses: actions/cache@v3 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.python-setup.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + - name: Install Poetry + run: pip install poetry - name: Install dependencies - if: steps.dependencies-cache.outputs.cache-hit != 'true' - run: | - poetry install --with dev --no-interaction --no-root + run: poetry install - # TODO: add linting / black / flake8 - # - name: Lint with flake8 - # run: | - # source .venv/bin/activate - # # stop the build if there are Python syntax errors or undefined names - # flake8 ./evaluation_function --count --select=E9,F63,F7,F82 --show-source --statistics - # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 ./evaluation_function --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Lint with flake8 + run: | + poetry run flake8 ./evaluation_function --count --select=E9,F63,F7,F82 --show-source --statistics + poetry run flake8 ./evaluation_function --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Run tests if: always() - run: | - source .venv/bin/activate - pytest --junit-xml=./reports/pytest.xml --tb=auto -v + run: poetry run pytest --junit-xml=./reports/pytest.xml --tb=auto -v - name: Upload test results uses: actions/upload-artifact@v4 if: always() with: - name: test-results + name: test-results-${{ matrix.python-version }} path: ./reports/pytest.xml if-no-files-found: warn deploy: needs: test - uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@deploy-request + permissions: + contents: write + packages: write + id-token: write + uses: lambda-feedback/evaluation-function-workflows/.github/workflows/deploy.yml@main with: template-repository-name: "lambda-feedback/evaluation-function-boilerplate-python" build-platforms: "aws" diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml new file mode 100644 index 0000000..700fe7b --- /dev/null +++ b/.github/workflows/test-lint.yml @@ -0,0 +1,49 @@ +name: Test and Lint + +on: + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + checks: write + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + id: python-setup + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry + run: pip install poetry + + - name: Install dependencies + run: poetry install + + - name: Lint with flake8 + run: | + poetry run flake8 ./evaluation_function --count --select=E9,F63,F7,F82 --show-source --statistics + poetry run flake8 ./evaluation_function --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Run tests + if: always() + run: poetry run pytest --junit-xml=./reports/pytest.xml --tb=auto -v + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.python-version }} + path: ./reports/pytest.xml + if-no-files-found: warn \ No newline at end of file