Merge pull request #1026 from autonomys/feat/add-decompression-support

Add decompression support
autonomys · Dec 12, 2024 · 0e2670c · 0e2670c
2 parents 7559215 + 0d4e311
commit 0e2670c
Show file tree

Hide file tree

Showing 8 changed files with 107 additions and 5 deletions.
diff --git a/explorer/gql/graphql.tsx b/explorer/gql/graphql.tsx
@@ -18257,7 +18257,7 @@ export type GetCidQueryVariables = Exact<{
 }>;
 
 
-export type GetCidQuery = { __typename?: 'query_root', files_metadata: Array<{ __typename?: 'files_metadata', chunk?: { __typename?: 'files_chunks', data?: string | null } | null, metadata_cids: Array<{ __typename?: 'files_metadata_cids', chunk?: { __typename?: 'files_chunks', data?: string | null } | null }> }>, files_folders: Array<{ __typename?: 'files_folders', chunk?: { __typename?: 'files_chunks', data?: string | null } | null, folder_cids: Array<{ __typename?: 'files_folder_cids', chunk?: { __typename?: 'files_chunks', data?: string | null } | null }> }>, files_files: Array<{ __typename?: 'files_files', chunk?: { __typename?: 'files_chunks', data?: string | null } | null, file_cids: Array<{ __typename?: 'files_file_cids', chunk?: { __typename?: 'files_chunks', data?: string | null } | null }> }> };
+export type GetCidQuery = { __typename?: 'query_root', files_metadata: Array<{ __typename?: 'files_metadata', chunk?: { __typename?: 'files_chunks', data?: string | null, uploadOptions?: string | null } | null, metadata_cids: Array<{ __typename?: 'files_metadata_cids', chunk?: { __typename?: 'files_chunks', data?: string | null } | null }> }>, files_folders: Array<{ __typename?: 'files_folders', chunk?: { __typename?: 'files_chunks', data?: string | null, uploadOptions?: string | null } | null, folder_cids: Array<{ __typename?: 'files_folder_cids', chunk?: { __typename?: 'files_chunks', data?: string | null } | null }> }>, files_files: Array<{ __typename?: 'files_files', chunk?: { __typename?: 'files_chunks', data?: string | null, uploadOptions?: string | null } | null, file_cids: Array<{ __typename?: 'files_file_cids', chunk?: { __typename?: 'files_chunks', data?: string | null } | null }> }> };
 
 export type FoldersQueryVariables = Exact<{
   limit: Scalars['Int']['input'];
@@ -21854,6 +21854,7 @@ export const GetCidDocument = gql`
   files_metadata(where: {id: {_eq: $cid}}) {
     chunk {
       data
+      uploadOptions: upload_options
     }
     metadata_cids {
       chunk {
@@ -21864,6 +21865,7 @@ export const GetCidDocument = gql`
   files_folders(where: {id: {_eq: $cid}}) {
     chunk {
       data
+      uploadOptions: upload_options
     }
     folder_cids {
       chunk {
@@ -21874,6 +21876,7 @@ export const GetCidDocument = gql`
   files_files(where: {id: {_eq: $cid}}) {
     chunk {
       data
+      uploadOptions: upload_options
     }
     file_cids {
       chunk {

diff --git a/explorer/package.json b/explorer/package.json
@@ -80,6 +80,7 @@
     "web-vitals": "^3.5.2",
     "xlsx": "^0.18.5",
     "yup": "^0.32.11",
+    "zlibjs": "^0.3.1",
     "zod": "^3.22.4",
     "zustand": "^4.5.2"
   },

diff --git a/explorer/src/components/Storage/Files/FilePreview.tsx b/explorer/src/components/Storage/Files/FilePreview.tsx
@@ -83,7 +83,10 @@ export const FilePreview: FC<Props> = ({ cid }) => {
         try {
           return (
             <div className='mb-4 w-full break-all rounded-lg border border-purpleLight bg-purpleLight p-4 shadow dark:border-none dark:bg-white/10'>
-              <Arguments args={JSON.parse(Buffer.from(rawData.data).toString('utf-8'))} />
+              <Arguments
+                args={JSON.parse(Buffer.from(rawData.data).toString('utf-8'))}
+                collapseAtEntry={25}
+              />
             </div>
           )
         } catch {

diff --git a/explorer/src/components/Storage/Files/query.ts b/explorer/src/components/Storage/Files/query.ts
@@ -45,6 +45,7 @@ export const QUERY_CID = gql`
     files_metadata(where: { id: { _eq: $cid } }) {
       chunk {
         data
+        uploadOptions: upload_options
       }
       metadata_cids {
         chunk {
@@ -55,6 +56,7 @@ export const QUERY_CID = gql`
     files_folders(where: { id: { _eq: $cid } }) {
       chunk {
         data
+        uploadOptions: upload_options
       }
       folder_cids {
         chunk {
@@ -65,6 +67,7 @@ export const QUERY_CID = gql`
     files_files(where: { id: { _eq: $cid } }) {
       chunk {
         data
+        uploadOptions: upload_options
       }
       file_cids {
         chunk {

diff --git a/explorer/src/components/common/Arguments.tsx b/explorer/src/components/common/Arguments.tsx
@@ -4,6 +4,7 @@ import ReactJson from 'react-json-view'
 type Props = {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   args: any
+  collapseAtEntry?: number
 }
 
 const theme = {
@@ -25,7 +26,7 @@ const theme = {
   base0F: 'inherit',
 }
 
-export const Arguments: FC<Props> = ({ args }) => {
+export const Arguments: FC<Props> = ({ args, collapseAtEntry = 5 }) => {
   return (
     <div data-testid='testJsonDisplay' className='text-grayDark dark:text-white'>
       <ReactJson
@@ -34,7 +35,9 @@ export const Arguments: FC<Props> = ({ args }) => {
         theme={theme}
         collapseStringsAfterLength={100}
         shouldCollapse={(field) => {
-          return field.type === 'object' && Object.entries(field.src).length > 5 ? true : false
+          return field.type === 'object' && Object.entries(field.src).length > collapseAtEntry
+            ? true
+            : false
         }}
       />
     </div>

diff --git a/explorer/src/utils/file.ts b/explorer/src/utils/file.ts
@@ -1,3 +1,6 @@
+import { GetCidQuery } from 'gql/graphql'
+import { Zlib } from 'zlibjs/bin/zlib_and_gzip.min.js'
+
 export const detectFileType = async (arrayBuffer: ArrayBuffer): Promise<string> => {
   const bytes = [...new Uint8Array(arrayBuffer.slice(0, 4))]
     .map((byte) => byte.toString(16).padStart(2, '0'))
@@ -88,11 +91,28 @@ const extractFileDataByType = (data: any, type: 'file' | 'folder' | 'metadata')
       index++
     }
   }
+  try {
+    if (data['files_' + type + 's'][0].chunk?.uploadOptions) {
+      const options = JSON.parse(data['files_' + type + 's'][0].chunk?.uploadOptions)
+      if (options.compression.algorithm === 'ZLIB') {
+        const inflate = new Zlib.Inflate(new Uint8Array(dataArrayBuffer), {
+          index: 0,
+          bufferSize: 1024,
+          bufferType: Zlib.Inflate.BufferType.BLOCK,
+          resize: true,
+          verify: true,
+        })
+        dataArrayBuffer = inflate.decompress()
+      }
+    }
+  } catch (error) {
+    console.error('Error decompressing data:', error)
+  }
   return { rawData, dataArrayBuffer }
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const extractFileData = (data: any) => {
+export const extractFileData = (data: GetCidQuery) => {
   if (data.files_files.length > 0) return extractFileDataByType(data, 'file')
   else if (data.files_folders.length > 0) return extractFileDataByType(data, 'folder')
   return extractFileDataByType(data, 'metadata')

diff --git a/explorer/yarn.lock b/explorer/yarn.lock
@@ -10301,6 +10301,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/zen-observable/-/zen-observable-0.8.15.tgz#96415c512d8e3ffd920afd3889604e30b9eaac15"
   integrity sha512-PQ2PC7R9rslx84ndNBZB/Dkv8V8fZEpk83RLgXtYd0fwUgEjseMn1Dgajh2x6S8QbZAFa9p2qVCEuYZNgve0dQ==
 
+zlibjs@^0.3.1:
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/zlibjs/-/zlibjs-0.3.1.tgz#50197edb28a1c42ca659cc8b4e6a9ddd6d444554"
+  integrity sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==
+
 zod@^3.22.4:
   version "3.22.4"
   resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.4.tgz#f31c3a9386f61b1f228af56faa9255e845cf3fff"

diff --git a/explorer/zlibjs.d.ts b/explorer/zlibjs.d.ts
@@ -0,0 +1,64 @@
+declare module 'zlibjs/bin/zlib_and_gzip.min.js' {
+  export namespace Zlib {
+    enum CompressionType {
+      NONE = 0,
+      FIXED = 1,
+      DYNAMIC = 2,
+    }
+
+    export namespace Gzip {
+      interface Options {
+        deflateOptions: {
+          compressionType: Zlib.CompressionType
+        }
+        flags: {
+          fname: boolean // use filename?
+          comment: boolean // use comment?
+          fhcrc: boolean // use file checksum?
+        }
+        filename: string // filename
+        comment: string
+      }
+    }
+
+    export namespace Deflate {
+      interface Options {
+        compressionType: CompressionType
+      }
+    }
+
+    export namespace Inflate {
+      enum BufferType {
+        BLOCK = 0,
+        ADAPTIVE = 1,
+      }
+      interface Options {
+        index: number // start position in input buffer
+        bufferSize: number // initial output buffer size
+        bufferType: Zlib.Inflate.BufferType // buffer expantion type
+        resize: boolean // resize buffer(ArrayBuffer) when end of decompression (default: false)
+        verify: boolean // verify decompression result (default: false)
+      }
+    }
+
+    export class Gzip {
+      constructor(data: Array<number> | Uint8Array, options: Gzip.Options)
+      public compress(): Uint8Array
+    }
+
+    export class Gunzip {
+      constructor(data: Array<number> | Uint8Array)
+      public decompress(): Uint8Array
+    }
+
+    export class Deflate {
+      constructor(data: Array<number> | Uint8Array, options: Deflate.Options)
+      public compress(): Uint8Array
+    }
+
+    export class Inflate {
+      constructor(data: Array<number> | Uint8Array, options: Inflate.Options)
+      public decompress(): Uint8Array
+    }
+  }
+}