feat(weavedrive): add l2 header access to weavedrive

extensions/weavedrive/client/main.lua

@@ -45,4 +45,18 @@ function drive.getData(txId)
   return contents
 end
 
+function drive.getDataItem(txId)
+  local file = io.open('/tx2/' .. txId)
+  if not file then
+    return nil, "File not found!"
+  end
+  if file == 'GQL Not Found!' then
+    return nil, 'GraphQL not Found!'
+  end
+  local contents = file:read(
+    file:seek('end')
+  )
+  file:close()
+  return contents
+end
 return drive

extensions/weavedrive/package.json

@@ -4,7 +4,7 @@
   "main": "./dist/index.cjs",
   "license": "MIT",
   "devDependencies": {
-    "@permaweb/ao-loader": "^0.0.36",
+    "@permaweb/ao-loader": "^0.0.37",
     "esbuild": "^0.21.1"
   }
 }

extensions/weavedrive/src/index.js

@@ -127,35 +127,121 @@ module.exports = function weaveDrive(mod, FS) {
       var stream = FS.open('/tx/' + id, 'r');
       return stream;
     },
+    async createDataItemTxHeader(id) {
+      const gqlQuery = this.gqlQuery
+      async function toAddress(owner) {
+        return Arweave.utils.bufferTob64Url(
+          await Arweave.crypto.hash(Arweave.utils.b64UrlToBuffer(owner))
+        );
+      }
+      async function retry(x) {
+        return new Promise(r => {
+          setTimeout(function () {
+            r(gqlQuery(`/tx/${id}`))
+          }, x * 10000)
+        })
+      }
+
+      const gqlExists = await this.gqlExists()
+      if (!gqlExists) {
+        return 'GQL Not Found!'
+      }
+      var GET_TRANSACTION_QUERY = `
+        query GetTransactions ($transactionIds: [ID!]!) {
+          transactions(ids: $transactionIds) {
+            edges {
+              node {
+                id
+                anchor
+                data {
+                  size
+                }
+                signature
+                recipient 
+                owner {
+                  address 
+                  key
+                }
+                fee {
+                  ar 
+                  winston
+                }
+                quantity {
+                  winston
+                  ar
+                }
+                tags {
+                  name 
+                  value 
+                }
+                bundledIn {
+                  id
+                }
+                block { 
+                  id
+                  timestamp
+                  height
+                  previous
+                }
+              }
+            }
+          }
+        }`
+      var variables = { transactionIds: [id] }
+      // todo: add a bunch of retries
+      var result = await this.gqlQuery(GET_TRANSACTION_QUERY, variables)
+        .then(res => !res.ok ? retry(1) : res)
+        .then(res => !res.ok ? retry(2) : res)
+        .then(res => !res.ok ? retry(3) : res)
+        .then(res => !res.ok ? retry(4) : res)
+        .then(res => res.json())
+        .then(res => res.data.transactions.edges[0].node)
+        .then(async entry => {
+          return { 
+            format: 3,
+            ...entry
+          }
+        })
+        .then(x => JSON.stringify(x));
+
+        var node = FS.createDataFile('/', 'tx2/' + id, result, true, false);
+        var stream = FS.open('/tx2/' + id, 'r');
 
+        return stream;
+    },
     async open(filename) {
       const pathCategory = filename.split('/')[1];
       const id = filename.split('/')[2];
       console.log("JS: Opening ID: ", id);
       if (pathCategory === 'tx') {
         FS.createPath('/', 'tx', true, false);
         if (FS.analyzePath(filename).exists) {
-          for (var i = 0; i < FS.streams.length; i++) {
-            if (FS.streams[i].node.name === id) {
-              return FS.streams[i].fd;
-            }
-          }
-          return 0;
+          var stream = FS.open(filename, 'r');
+          if (stream.fd) return stream.fd
+          return 0
         } else {
           const stream = await this.createTxHeader(id);
           return stream.fd;
         }
 
       }
+      if (pathCategory === 'tx2') {
+        FS.createPath('/', 'tx2', true, false);
+        if (FS.analyzePath(filename).exists) {
+          var stream = FS.open(filename, 'r');
+          if (stream.fd) return stream.fd
+          return 0
+        } else {
+          const stream = await this.createDataItemTxHeader(id);
+          return stream.fd;
+        }
+      }
       if (pathCategory === 'block') {
         FS.createPath('/', 'block', true, false);
         if (FS.analyzePath(filename).exists) {
-          for (var i = 0; i < FS.streams.length; i++) {
-            if (FS.streams[i].node.name === id) {
-              return FS.streams[i].fd;
-            }
-          }
-          return 0;
+          var stream = FS.open(filename, 'r');
+          if (stream.fd) return stream.fd
+          return 0
         } else {
           const stream = await this.createBlockHeader(id);
           return stream.fd;
@@ -164,12 +250,8 @@ module.exports = function weaveDrive(mod, FS) {
 
       if (pathCategory === 'data') {
         if (FS.analyzePath(filename).exists) {
-          for (var i = 0; i < FS.streams.length; i++) {
-            if (FS.streams[i].node.name === id) {
-              //console.log("JS: Found file: ", filename, " fd: ", FS.streams[i].fd);
-              return FS.streams[i].fd;
-            }
-          }
+          var stream = FS.open(filename, 'r');
+          if (stream.fd) return stream.fd
           console.log("JS: File not found: ", filename);
           return 0;
         }
@@ -189,7 +271,6 @@ module.exports = function weaveDrive(mod, FS) {
         return 0;
       }
     },
-
     async read(fd, raw_dst_ptr, raw_length) {
 
       // Note: The length and dst_ptr are 53 bit integers in JS, so this _should_ be ok into a large memspace.
@@ -468,6 +549,61 @@ module.exports = function weaveDrive(mod, FS) {
       const results = await mod.arweave.api.post('/graphql', query);
       const json = JSON.parse(results)
       return json.data.transactions.edges.length > 0
+    },
+
+    async gqlExists() {
+      const query = `query {
+        transactions(
+          first: 1
+        ) {
+          pageInfo {
+            hasNextPage
+          }
+        }
+      }
+      `
+
+      const gqlExists = await this.gqlQuery(query, {}).then((res) => res.ok)
+      return gqlExists
+    },
+
+    async gqlQuery(query, variables) {
+      let urlList = null
+      const headers = new Headers({})
+      headers.append("content-type", "application/json")
+      if (mod.ARWEAVE.includes(',')) {
+        urlList = mod.ARWEAVE.split(',').map(url => url.trim())
+      }
+      if (urlList && urlList.length > 0) {
+        /**
+         * Try a list of gateways instead of a single one
+         */
+        for (const url of urlList) {
+          const response = await fetch(`${url}/graphql`, {
+            method: 'POST',
+            body: JSON.stringify({ query, variables }),
+            headers
+          })
+          if (response.ok) {
+            return response
+          }
+        }
+        /**
+         * None succeeded so fall back to mod.ARWEAVE so that
+         * if this fails we return a proper error response
+         */
+        return await fetch(`${mod.ARWEAVE}/graphql`, {
+          method: 'POST',
+          body: JSON.stringify({ query, variables }),
+          headers
+        })
+      } else {
+        return await fetch(`${mod.ARWEAVE}/graphql`, {
+          method: 'POST',
+          body: JSON.stringify({ query, variables }),
+          headers
+        })
+      }
     }
   }
-}
+}

extensions/weavedrive/test/index.test.js

@@ -3,7 +3,7 @@ const fs = require('fs')
 const { test } = require('node:test')
 const assert = require('assert')
 const weaveDrive = require('../src/index.js')
-const wasm = fs.readFileSync('./aosqlite.wasm')
+const wasm = fs.readFileSync('./process.wasm')
 const bootLoaderWasm = fs.readFileSync('./bootloader.wasm')
 
 let memory = null
@@ -14,7 +14,8 @@ const Module = {
   Tags: [
     { name: 'Data-Protocol', value: 'ao'},
     { name: 'Variant', value: 'ao.TN.1'},
-    { name: 'Type', value: 'Module'}
+    { name: 'Type', value: 'Module'},
+    { name: 'Authority', value: 'PROCESS' }
   ]
 }
 
@@ -25,7 +26,9 @@ const Process = {
     { name: 'Data-Protocol', value: 'ao' },
     { name: 'Variant', value: 'ao.TN.1' },
     { name: 'Type', value: 'Process' },
-    { name: 'Extension', value: 'WeaveDrive' }
+    { name: 'Extension', value: 'WeaveDrive' },
+    { name: 'Module', value: 'MODULE' },
+    { name: 'Authority', value: 'PROCESS' },
   ]
 }
 
@@ -82,11 +85,9 @@ test('read block', async () => {
 `
   }, { Process, Module })
   memory = result.Memory
-  console.log(result.Output.data.output)
-  assert.equal(result.Output.data.output, '63')
+  assert.equal(result.Output.data, '63')
 })
 
-
 test('read tx', async () => {
   const handle = await AoLoader(wasm, options)
   const result = await handle(memory, {
@@ -109,7 +110,6 @@ return results
     `
   }, { Process, Module })
   memory = result.Memory
-  console.log(result.Output.data.output)
   assert.ok(true)
 })
 
@@ -119,7 +119,7 @@ test('read twice', async function () {
     ...Msg,
     Data: `
 local drive = require('WeaveDrive')
-function getTxs(txs)
+function getTxs()
   local results = {}
   local txs = drive 
     .getBlock('1439783').txs
@@ -135,15 +135,13 @@ function getTxs(txs)
   end
   return results
 end
-local block = drive.getBlock('1439782')
-local results = getTxs(block.txs) 
-local results2 = getTxs(block.txs)
-
+local results = getTxs() 
+local results2 = getTxs()
 return require('json').encode({ A = #results, B = #results2 }) 
     `
   }, { Process, Module })
   memory = result.Memory
-  const res = JSON.parse(result.Output.data.output)
+  const res = JSON.parse(result.Output.data)
 
   assert.equal(res.A, res.B)
 })
@@ -184,10 +182,44 @@ return results
     `
   }, { Process, Module })
   memory = result.Memory
-  console.log(result)
   assert.ok(true)
 })
 
+test('read data item tx', async () => {
+  const handle = await AoLoader(wasm, options)
+  const result = await handle(memory, {
+    ...Msg,
+    Data: `
+local results = {}
+local drive = require('WeaveDrive')
+local result = drive.getDataItem('Jy_AFHfmxVsrtJoxeJZfq9dx_ES730a7uO2lyYtO6uU')
+
+return result
+    `
+  }, { Process, Module })
+  const data = JSON.parse(result.Output.data)
+  assert.equal(data.format, 3)
+  assert.equal(data.id, 'Jy_AFHfmxVsrtJoxeJZfq9dx_ES730a7uO2lyYtO6uU')
+  assert.equal(data.block.height, 1290333)
+})
+
+test('read data item tx, no gql', async () => {
+  const handle = await AoLoader(wasm, {
+    ...options,
+    ARWEAVE: 'https://example.com'
+  })
+  const result = await handle(memory, {
+    ...Msg,
+    Data: `
+local results = {}
+local drive = require('WeaveDrive')
+local result = drive.getDataItem('Jy_AFHfmxVsrtJoxeJZfq9dx_ES730a7uO2lyYtO6uU')
+return result
+    `
+  }, { Process, Module })
+  memory = result.Memory
+  assert.equal(result.Output.data, '')
+})
 
 let memoryBootLoader = null