New GetResult Arguments (filter, sort, sample, columns) (#40)

- filter out some rows before retrieve the results
- specify the output columns
- get a sample of the results
- sort the results

src/api/client.ts

@@ -57,10 +57,11 @@ export class DuneClient {
    * @returns Execution Results
    */
   async runQuery(args: RunQueryArgs): Promise<ResultsResponse> {
-    const { queryId, params, opts } = args;
+    const { queryId, opts } = args;
+    args.limit = opts?.batchSize || args.limit;
     const { state, execution_id } = await this._runInner(
       queryId,
-      params,
+      args,
       opts?.pingFrequency,
     );
     if (state === ExecutionState.COMPLETED) {
@@ -86,18 +87,16 @@ export class DuneClient {
    * @returns Execution Results as CSV
    */
   async runQueryCSV(args: RunQueryArgs): Promise<ExecutionResponseCSV> {
-    const { queryId, params, opts } = args;
+    const { queryId, opts } = args;
+    args.limit = opts?.batchSize || args.limit;
     const { state, execution_id } = await this._runInner(
       queryId,
-      params,
+      args,
       opts?.pingFrequency,
     );
     if (state === ExecutionState.COMPLETED) {
-      // we can't assert that the execution ids agree here,
-      // so we use max age hours as a "safe guard"
-      return this.exec.getResultCSV(execution_id, {
-        query_parameters: params?.query_parameters,
-      });
+      // we can't assert that the execution ids agree here!
+      return this.exec.getLastResultCSV(queryId, args);
     } else {
       const message = `refresh (execution ${execution_id}) yields incomplete terminal state ${state}`;
       // TODO - log the error in constructor
@@ -114,19 +113,17 @@ export class DuneClient {
    * @returns Latest execution results for the given parameters.
    */
   async getLatestResult(args: RunQueryArgs): Promise<ResultsResponse> {
-    const { queryId, params, opts } = args;
+    const { queryId, opts } = args;
+    args.limit = opts?.batchSize || args.limit;
     const lastestResults = await this.exec.getLastExecutionResults(
       queryId,
-      {
-        query_parameters: params?.query_parameters,
-        limit: opts?.batchSize,
-      },
+      args,
       opts?.maxAgeHours,
     );
     let results: ResultsResponse;
     if (lastestResults.isExpired) {
       log.info(logPrefix, `results expired, re-running query.`);
-      results = await this.runQuery({ queryId, params, opts });
+      results = await this.runQuery(args);
     } else {
       results = lastestResults.results;
     }
@@ -140,24 +137,19 @@ export class DuneClient {
    * @param outFile - location to save CSV.
    */
   async downloadCSV(args: RunQueryArgs, outFile: string): Promise<void> {
-    const { queryId, params, opts } = args;
+    const { queryId, opts } = args;
+    args.limit = opts?.batchSize || args.limit;
     const { isExpired } = await this.exec.getLastExecutionResults(
       queryId,
-      {
-        query_parameters: params?.query_parameters,
-        limit: opts?.batchSize,
-      },
+      args,
       args.opts?.maxAgeHours,
     );
     let results: Promise<ExecutionResponseCSV>;
     if (isExpired) {
       results = this.runQueryCSV(args);
     } else {
       // TODO (user cost savings): transform the lastResults into CSV instead of refetching
-      results = this.exec.getLastResultCSV(args.queryId, {
-        query_parameters: args.params?.query_parameters,
-        limit: args.opts?.batchSize,
-      });
+      results = this.exec.getLastResultCSV(args.queryId, args);
     }
     // Wait for the results promise to resolve and then write the CSV data to the specified outFile
     const csvData = (await results).data;
@@ -174,17 +166,17 @@ export class DuneClient {
    * @returns {Promise<ResultsResponse>}
    */
   public async runSql(args: RunSqlArgs): Promise<ResultsResponse> {
-    const { name, query_sql, params, isPrivate, archiveAfter, opts } = args;
+    const { name, query_sql, isPrivate, query_parameters, archiveAfter } = args;
     const queryId = await this.query.createQuery({
       name: name ? name : "API Query",
       query_sql,
-      query_parameters: params?.query_parameters,
+      query_parameters,
       is_private: isPrivate,
     });
     let results: ResultsResponse;
 
     try {
-      results = await this.runQuery({ queryId, params, opts });
+      results = await this.runQuery({ queryId, ...args });
     } finally {
       if (archiveAfter) {
         this.query.archiveQuery(queryId);
@@ -228,7 +220,7 @@ export class DuneClient {
   ): Promise<ResultsResponse> {
     return this.runQuery({
       queryId: queryID,
-      params: { query_parameters: parameters },
+      query_parameters: parameters,
       opts: { pingFrequency },
     });
   }

src/api/execution.ts

@@ -11,15 +11,15 @@ import {
   ExecutionParams,
   ExecutionPerformance,
   GetResultParams,
+  validateAndBuildGetResultParams,
 } from "../types";
 import log from "loglevel";
-import { ageInHours, logPrefix, withDefaults } from "../utils";
+import { ageInHours, logPrefix } from "../utils";
 import { Router } from "./router";
 import {
   DEFAULT_GET_PARAMS,
   DUNE_CSV_NEXT_OFFSET_HEADER,
   DUNE_CSV_NEXT_URI_HEADER,
-  MAX_NUM_ROWS_PER_BATCH,
   THREE_MONTHS_IN_HOURS,
 } from "../constants";
 
@@ -95,7 +95,7 @@ export class ExecutionAPI extends Router {
   ): Promise<ResultsResponse> {
     const response: ResultsResponse = await this._get(
       `execution/${executionId}/results`,
-      withDefaults(params, { limit: MAX_NUM_ROWS_PER_BATCH }),
+      validateAndBuildGetResultParams(params),
     );
     log.debug(logPrefix, `get_result response ${JSON.stringify(response)}`);
     return response as ResultsResponse;
@@ -114,7 +114,7 @@ export class ExecutionAPI extends Router {
   ): Promise<ExecutionResponseCSV> {
     const response = await this._get<Response>(
       `execution/${executionId}/results/csv`,
-      params,
+      validateAndBuildGetResultParams(params),
       true,
     );
     log.debug(logPrefix, `get_result response ${JSON.stringify(response)}`);
@@ -137,7 +137,7 @@ export class ExecutionAPI extends Router {
     // The first bit might only return a page.
     const results = await this._get<ResultsResponse>(
       `query/${queryId}/results`,
-      withDefaults(params, { limit: MAX_NUM_ROWS_PER_BATCH }),
+      validateAndBuildGetResultParams(params),
     );
     const lastRun: Date = results.execution_ended_at!;
     const maxAge = expiryAgeHours;
@@ -157,7 +157,7 @@ export class ExecutionAPI extends Router {
   ): Promise<ExecutionResponseCSV> {
     const response = await this._get<Response>(
       `query/${queryId}/results/csv`,
-      withDefaults(params, { limit: MAX_NUM_ROWS_PER_BATCH }),
+      validateAndBuildGetResultParams(params),
       true,
     );
     return this._fetchEntireResultCSV(await this.buildCSVResponse(response));

src/api/router.ts

@@ -120,7 +120,7 @@ export class Router {
       const searchParams = new URLSearchParams(payloadSearchParams(payload)).toString();
       queryParams = `?${searchParams}`;
     }
-
+    log.debug("Final request URL", url + queryParams);
     const response = fetch(url + queryParams, requestData);
     if (raw) {
       return response as T;

src/constants.ts

@@ -1,10 +1,9 @@
-import { GetResultParams } from "./types/requestPayload";
+import { GetResultParams } from "./types/requestArgs";
 
 // Seconds between checking execution status
 export const POLL_FREQUENCY_SECONDS = 1;
 // This is the expiry time on old query results.
 export const THREE_MONTHS_IN_HOURS = 2191;
-
 // Headers used for pagination in CSV results
 export const DUNE_CSV_NEXT_URI_HEADER = "x-dune-next-uri";
 export const DUNE_CSV_NEXT_OFFSET_HEADER = "x-dune-next-offset";

src/types/index.ts

@@ -1,6 +1,5 @@
-export * from "./client";
 export * from "./error";
 export * from "./query";
 export * from "./queryParameter";
-export * from "./requestPayload";
+export * from "./requestArgs";
 export * from "./response";

src/types/query.ts

@@ -6,31 +6,29 @@ import { QueryParameter } from "./queryParameter";
  * https://docs.dune.com/api-reference/queries/endpoint/read#example-return
  */
 export interface DuneQuery {
-  /// ID of the created query.
-  query_id: number;
+  /// Description of the query.
+  description: string;
+  /// Indicates if the query is archived.
+  /// Note: This is as close as a user can get to deleting a query.
+  is_archived: boolean;
+  /// Indicates if the query is private.
+  is_private: boolean;
+  /// Indicates if the query is unsaved.
+  is_unsaved: boolean;
   /// Name of the query.
   name: string;
-  /// Query description.
-  description: string;
-  /// Tags associated with the query.
-  tags: string[];
-  /// query revision version.
-  version: number;
+  /// Dune user who owns the query.
+  owner: string;
   /// Parameters with their names and default values.
   parameters: QueryParameter[];
-  /// Query engine query was created for.
-  /// All legacy engines have been deprecated,
-  /// so this is essentially no longer relevant.
+  /// The query engine used to execute the query.
   query_engine: string;
+  /// Unique identifier of the query.
+  query_id: number;
   /// Raw SQL of the query.
   query_sql: string;
-  /// whether or not the query is private.
-  is_private: boolean;
-  /// whether or not the query is archived.
-  /// Note: This is as close as a user can get to deleting a query.
-  is_archived: boolean;
-  /// whether or not the query is unsaved.
-  is_unsaved: boolean;
-  /// Dune user who owns the query.
-  owner: string;
+  /// Tags associated with the query.
+  tags: string[];
+  /// Version of the query.
+  version: number;
 }