Merge remote-tracking branch 'upstream/master'

* upstream/master: chore: update the link to the interop dashboard [skip ci] chore: parameterise s3 build cache setup (#465) chore: Create funding.json
chore: update the link to the interop dashboard [skip ci]
2024-09-12 18:51:12 +02:00 · 2024-09-11 08:25:10 +00:00 · 2024-09-11 10:01:42 +02:00 · 2024-09-05 13:49:25 -04:00 · 2024-08-23 20:52:39 +00:00 · 2024-08-22 15:00:11 +02:00
9 changed files with 68 additions and 40 deletions
@@ -86,6 +86,11 @@ runs:

    - name: Load cache and build
      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      env:
+        AWS_BUCKET: ${{ inputs.s3-cache-bucket }}
+        AWS_REGION: ${{ inputs.aws-region }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
      run: npm run cache -- load
      shell: bash

@@ -78,6 +78,11 @@ runs:

    - name: Load cache and build
      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      env:
+        AWS_BUCKET: ${{ inputs.s3-cache-bucket }}
+        AWS_REGION: ${{ inputs.aws-region }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
      run: npm run cache -- load
      shell: bash

@@ -78,6 +78,11 @@ runs:

    - name: Load cache and build
      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      env:
+        AWS_BUCKET: ${{ inputs.s3-cache-bucket }}
+        AWS_REGION: ${{ inputs.aws-region }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
      run: npm run cache -- load
      shell: bash

@@ -28,7 +28,7 @@ jobs:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/run-interop-hole-punch-test
        with:
-          s3-cache-bucket: libp2p-by-tf-aws-bootstrap
-          s3-access-key-id: ${{ vars.S3_AWS_ACCESS_KEY_ID }}
-          s3-secret-access-key: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
+          s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }}
+          s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }}
+          s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }}
          worker-count: 16
@@ -18,12 +18,12 @@ jobs:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/run-transport-interop-test
        with:
-          s3-cache-bucket: libp2p-by-tf-aws-bootstrap
-          s3-access-key-id: ${{ vars.S3_AWS_ACCESS_KEY_ID }}
-          s3-secret-access-key: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
+          s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }}
+          s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }}
+          s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }}
          worker-count: 16
  build-without-secrets:
-    runs-on: ubuntu-latest
+    runs-on: ['self-hosted', 'linux', 'x64', '4xlarge'] # https://github.com/pl-strflt/tf-aws-gh-runner/blob/main/runners.tf
    steps:
      - uses: actions/checkout@v3
      # Purposely not using secrets to replicate how forks will behave.
@@ -1,6 +1,6 @@
 # Interoperability/end to end test-plans & performance benchmarking for libp2p

-[![Interop Dashboard](https://github.com/status-im/libp2p-test-plans/workflows/libp2p%20transport%20interop%20test/badge.svg?branch=master)](https://github.com/status-im/libp2p-test-plans/actions/runs/7043365318/attempts/1#summary-19169013615)
+[![Interop Dashboard](https://github.com/vacp2p/libp2p-test-plans/workflows/libp2p%20transport%20interop%20test/badge.svg?branch=master)](https://github.com/vacp2p/libp2p-test-plans/actions/runs/10508587062/attempts/1#summary-29112802717)

 [![Made by Protocol Labs](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://protocol.ai)

@@ -0,0 +1,5 @@
+{
+  "opRetro": {
+    "projectId": "0x966804cb492e1a4bde5d781a676a44a23d69aa5dd2562fa7a4f95bb606021c8b"
+  }
+}
@@ -1,8 +1,9 @@
-const AWS_BUCKET = process.env.AWS_BUCKET || 'libp2p-by-tf-aws-bootstrap';
+const AWS_BUCKET = process.env.AWS_BUCKET;
 const scriptDir = __dirname;

 import * as crypto from 'crypto';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 import * as child_process from 'child_process';
 import ignore, { Ignore } from 'ignore'
@@ -76,10 +77,14 @@ async function loadCacheOrBuild(dir: string, ig: Ignore) {
    if (mode == Mode.PushCache) {
        console.log("Pushing cache")
        try {
-            const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, {method: "HEAD"})
-            if (res.ok) {
+            if (!AWS_BUCKET) {
+                throw new Error("AWS_BUCKET not set")
+            }
+            try {
+                child_process.execSync(`aws s3 ls s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`)
                console.log("Cache already exists")
-            } else {
+            } catch (e) {
+                console.log("Cache doesn't exist", e)
                // Read image id from image.json
                const imageID = JSON.parse(fs.readFileSync(path.join(dir, 'image.json')).toString()).imageID;
                console.log(`Pushing cache for ${dir}: ${imageID}`)
@@ -96,18 +101,17 @@ async function loadCacheOrBuild(dir: string, ig: Ignore) {
        console.log("Loading cache")
        let cacheHit = false
        try {
-            // Check if the cache exists
-            const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, {method: "HEAD"})
-            if (res.ok) {
-                const dockerLoadedMsg = child_process.execSync(`curl https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz  | docker image load`).toString();
-                const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
-                if (loadedImageId) {
-                    console.log(`Cache hit for ${loadedImageId}`);
-                    fs.writeFileSync(path.join(dir, 'image.json'), JSON.stringify({imageID: loadedImageId}) + "\n");
-                    cacheHit = true
-                }
-            } else {
-                console.log("Cache not found")
+            if (!AWS_BUCKET) {
+                throw new Error("AWS_BUCKET not set")
+            }
+            const cachePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cache'))
+            const archivePath = path.join(cachePath, 'archive.tar.gz')
+            const dockerLoadedMsg = child_process.execSync(`aws s3 cp s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz ${archivePath} && docker image load -i ${archivePath}`).toString();
+            const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
+            if (loadedImageId) {
+                console.log(`Cache hit for ${loadedImageId}`);
+                fs.writeFileSync(path.join(dir, 'image.json'), JSON.stringify({imageID: loadedImageId}) + "\n");
+                cacheHit = true
            }
        } catch (e) {
            console.log("Cache not found:", e)
@@ -1,8 +1,9 @@
-const AWS_BUCKET = process.env.AWS_BUCKET || 'libp2p-by-tf-aws-bootstrap';
+const AWS_BUCKET = process.env.AWS_BUCKET;
 const scriptDir = __dirname;

 import * as crypto from 'crypto';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 import * as child_process from 'child_process';
 import ignore, { Ignore } from 'ignore'
@@ -65,10 +66,14 @@ switch (modeStr) {
            if (mode == Mode.PushCache) {
                console.log("Pushing cache")
                try {
-                    const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, { method: "HEAD" })
-                    if (res.ok) {
+                    if (!AWS_BUCKET) {
+                        throw new Error("AWS_BUCKET not set")
+                    }
+                    try {
+                        child_process.execSync(`aws s3 ls s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`)
                        console.log("Cache already exists")
-                    } else {
+                    } catch (e) {
+                        console.log("Cache doesn't exist", e)
                        // Read image id from image.json
                        const imageID = JSON.parse(fs.readFileSync(path.join(implFolder, 'image.json')).toString()).imageID;
                        console.log(`Pushing cache for ${impl}: ${imageID}`)
@@ -85,18 +90,17 @@ switch (modeStr) {
                console.log("Loading cache")
                let cacheHit = false
                try {
-                    // Check if the cache exists
-                    const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, { method: "HEAD" })
-                    if (res.ok) {
-                        const dockerLoadedMsg = child_process.execSync(`curl https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz  | docker image load`).toString();
-                        const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
-                        if (loadedImageId) {
-                            console.log(`Cache hit for ${loadedImageId}`);
-                            fs.writeFileSync(path.join(implFolder, 'image.json'), JSON.stringify({ imageID: loadedImageId }) + "\n");
-                            cacheHit = true
-                        }
-                    } else {
-                        console.log("Cache not found")
+                    if (!AWS_BUCKET) {
+                        throw new Error("AWS_BUCKET not set")
+                    }
+                    const cachePath = fs.mkdtempSync(path.join(os.tmpdir(), 'cache'))
+                    const archivePath = path.join(cachePath, 'archive.tar.gz')
+                    const dockerLoadedMsg = child_process.execSync(`aws s3 cp s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz ${archivePath} && docker image load -i ${archivePath}`).toString();
+                            const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
+                    if (loadedImageId) {
+                        console.log(`Cache hit for ${loadedImageId}`);
+                        fs.writeFileSync(path.join(implFolder, 'image.json'), JSON.stringify({ imageID: loadedImageId }) + "\n");
+                        cacheHit = true
                    }
                } catch (e) {
                    console.log("Cache not found:", e)
Author	SHA1	Message	Date
Alejandro Cabeza Romero	e7d9b15158	Merge remote-tracking branch 'upstream/master' * upstream/master: chore: update the link to the interop dashboard [skip ci] chore: parameterise s3 build cache setup (#465) chore: Create funding.json	2024-09-12 18:51:12 +02:00
github-actions[bot]	c164b24049	chore: update the link to the interop dashboard [skip ci]	2024-09-11 08:25:10 +00:00
Piotr GalarandAlex Potsides	e5e32c7c0a	chore: parameterise s3 build cache setup (#465 ) * chore: parameterise s3 build cache setup * Apply suggestions from code review Co-authored-by: Alex Potsides <alex@achingbrain.net> * fix: cache load/push * chore: pass creds when reading cache * fix: interact with aws only using the CLI * Update transport-interop.yml --------- Co-authored-by: Alex Potsides <alex@achingbrain.net>	2024-09-11 10:01:42 +02:00
Prithvi Shahi	e509958ad9	chore: Create funding.json Add funding.json for application to OP rPGF round 5	2024-09-05 13:49:25 -04:00
github-actions[bot]	40fde07559	chore: update the link to the interop dashboard [skip ci]	2024-08-23 20:52:39 +00:00
Álex	2748dde76a	Merge pull request #2 from vacp2p/update/upstream feat: update from upstream	2024-08-22 15:00:11 +02:00
github-actions[bot]	fa34779b3d	chore: update the link to the interop dashboard [skip ci]	2023-12-01 09:30:54 +00:00