-
Notifications
You must be signed in to change notification settings - Fork 206
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: integrate jobs.query and stateless query for faster queries #1337
Changes from 1 commit
c2a6841
344c2ce
ed4381c
af93d2b
1a3be39
0717724
970e08f
bce8c48
3757849
9a2cce5
7531a12
7141834
f6b19df
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -286,6 +286,12 @@ | |
* We will create a table with the correct schema, import the public CSV file | ||
* into that table, and query it for data. | ||
* | ||
* This client supports enabling query-related preview features via environmental | ||
* variables. By setting the environment variable QUERY_PREVIEW_ENABLED to the string | ||
* "TRUE", the client will enable preview features, though behavior may still be | ||
* controlled via the bigquery service as well. Currently, the feature(s) in scope | ||
* include: stateless queries (query execution without corresponding job metadata). | ||
* | ||
* @class | ||
* | ||
* See {@link https://cloud.google.com/bigquery/what-is-bigquery| What is BigQuery?} | ||
|
@@ -322,18 +328,19 @@ | |
export class BigQuery extends Service { | ||
location?: string; | ||
private _universeDomain: string; | ||
private _enableQueryPreview: boolean; | ||
|
||
createQueryStream(options?: Query | string): ResourceStream<RowMetadata> { | ||
// placeholder body, overwritten in constructor | ||
return new ResourceStream<RowMetadata>({}, () => {}); | ||
} | ||
|
||
getDatasetsStream(options?: GetDatasetsOptions): ResourceStream<Dataset> { | ||
// placeholder body, overwritten in constructor | ||
return new ResourceStream<Dataset>({}, () => {}); | ||
} | ||
|
||
getJobsStream(options?: GetJobsOptions): ResourceStream<Job> { | ||
// placeholder body, overwritten in constructor | ||
return new ResourceStream<Job>({}, () => {}); | ||
} | ||
|
@@ -379,6 +386,14 @@ | |
|
||
super(config, options); | ||
|
||
const QUERY_PREVIEW_ENABLED = process.env.QUERY_PREVIEW_ENABLED; | ||
this._enableQueryPreview = false; | ||
if (typeof QUERY_PREVIEW_ENABLED === 'string') { | ||
if (QUERY_PREVIEW_ENABLED.toUpperCase() === 'TRUE') { | ||
this._enableQueryPreview = true; | ||
} | ||
} | ||
|
||
this._universeDomain = universeDomain; | ||
this.location = options.location; | ||
/** | ||
|
@@ -1534,7 +1549,7 @@ | |
const parameterMode = is.array(params) ? 'positional' : 'named'; | ||
const queryParameters: bigquery.IQueryParameter[] = []; | ||
if (parameterMode === 'named') { | ||
const namedParams = params as {[param: string]: any}; | ||
for (const namedParameter in namedParams) { | ||
const value = namedParams[namedParameter]; | ||
let queryParameter; | ||
|
@@ -2177,7 +2192,7 @@ | |
|
||
options = extend({job}, queryOpts, options); | ||
if (res && res.jobComplete) { | ||
let rows: any = []; | ||
alvarowolfx marked this conversation as resolved.
Show resolved
Hide resolved
|
||
if (res.schema && res.rows) { | ||
rows = BigQuery.mergeSchemaWithRows_(res.schema, res.rows, { | ||
wrapIntegers: options.wrapIntegers!, // TODO: fix default value | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as above (re: not sure if this was moved from elsewhere or new code) but I wanted to ping in case the TODO isn't intentional. |
||
|
@@ -2195,6 +2210,9 @@ | |
job!.getQueryResults(options, callback as QueryRowsCallback); | ||
return; | ||
} | ||
if (options.timeoutMs) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can just delete, no need for the |
||
delete options.timeoutMs; | ||
} | ||
this.trace_('[runJobsQuery] job not complete'); | ||
job!.getQueryResults(options, callback as QueryRowsCallback); | ||
}); | ||
|
@@ -2273,6 +2291,9 @@ | |
if (req.maxResults) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Temporary workaround? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. gonna remove that |
||
req.jobCreationMode = 'JOB_CREATION_REQUIRED'; | ||
} | ||
if (!this._enableQueryPreview) { | ||
delete req.jobCreationMode; | ||
} | ||
const {parameterMode, params} = this.buildQueryParams_( | ||
queryObj.params, | ||
queryObj.types | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Might want to use
for (const namedParameter of Object.getOwnPropertyNames(namedParams))
or check eachnamedParameter
againstnamedParams.hasOwnProperty()
.