Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: switch timestamp representation to int64 usec #1332

Merged
merged 6 commits into from
Feb 21, 2024
Merged

feat: switch timestamp representation to int64 usec #1332

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
034d969
feat: switch timestamp representation to int64 usec
alvarowolfx Feb 7, 2024
32ff432
Merge branch 'main' into feat-lossless-timestamp
alvarowolfx Feb 12, 2024
c543b26
fix: properly handle number input
alvarowolfx Feb 16, 2024
fe67d57
fix: lint issues
alvarowolfx Feb 16, 2024
060bc78
Merge branch 'main' into feat-lossless-timestamp
alvarowolfx Feb 19, 2024
5bbc6d9
fix: handle negative unix timestamp values
alvarowolfx Feb 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Next Next commit
feat: switch timestamp representation to int64 usec
  • Loading branch information
@alvarowolfx
alvarowolfx committed Feb 7, 2024
commit 034d9698f2b9977216f22ac57e50b34d3bf56dd6
18 changes: 2 additions & 16 deletions src/bigquery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,17 +312,17 @@
export class BigQuery extends Service {
location?: string;

createQueryStream(options?: Query | string): ResourceStream<RowMetadata> {

Check warning on line 315 in src/bigquery.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<RowMetadata>({}, () => {});
}

getDatasetsStream(options?: GetDatasetsOptions): ResourceStream<Dataset> {

Check warning on line 320 in src/bigquery.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<Dataset>({}, () => {});
}

getJobsStream(options?: GetJobsOptions): ResourceStream<Job> {

Check warning on line 325 in src/bigquery.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<Job>({}, () => {});
}
Expand Down Expand Up @@ -2194,15 +2194,10 @@
if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) {
pd = new PreciseDate(value);
} else {
const floatValue = Number.parseFloat(value);
if (!Number.isNaN(floatValue)) {
pd = this.fromFloatValue_(floatValue);
} else {
pd = new PreciseDate(value);
}
pd = new PreciseDate(BigInt(value) * BigInt(1000));
}
} else {
pd = this.fromFloatValue_(value);
pd = new PreciseDate(value);
alvarowolfx marked this conversation as resolved.
Show resolved Hide resolved
}
// to keep backward compatibility, only converts with microsecond
// precision if needed.
Expand All @@ -2212,15 +2207,6 @@
this.value = new Date(pd.getTime()).toJSON();
}
}

fromFloatValue_(value: number): PreciseDate {
const secs = Math.trunc(value);
// Timestamps in BigQuery have microsecond precision, so we must
// return a round number of microseconds.
const micros = Math.trunc((value - secs) * 1e6 + 0.5);
const pd = new PreciseDate([secs, micros * 1000]);
return pd;
}
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/job.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@
location?: string;
projectId?: string;
getQueryResultsStream(
options?: QueryResultsOptions

Check warning on line 130 in src/job.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
): ResourceStream<RowMetadata> {
// placeholder body, overwritten in constructor
return new ResourceStream<RowMetadata>({}, () => {});
Expand Down Expand Up @@ -529,10 +529,10 @@
typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback =
typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;

const qs = extend(
{
location: this.location,
'formatOptions.useInt64Timestamp': true,
},
options
);
Expand Down
11 changes: 9 additions & 2 deletions src/table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@
bigQuery: BigQuery;
location?: string;
rowQueue?: RowQueue;
createReadStream(options?: GetRowsOptions): ResourceStream<RowMetadata> {

Check warning on line 223 in src/table.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<RowMetadata>({}, () => {});
}
Expand Down Expand Up @@ -1848,10 +1848,17 @@
callback!(null, rows, nextQuery, resp);
};

const qs = extend(
{
'formatOptions.useInt64Timestamp': true,
},
options
);

this.request(
{
uri: '/data',
qs: options,
qs,
},
(err, resp) => {
if (err) {
Expand All @@ -1860,7 +1867,7 @@
}
let nextQuery: GetRowsOptions | null = null;
if (resp.pageToken) {
nextQuery = Object.assign({}, options, {
nextQuery = Object.assign({}, qs, {
pageToken: resp.pageToken,
});
}
Expand Down
26 changes: 21 additions & 5 deletions test/job.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,10 @@ describe('BigQuery/Job', () => {

it('should optionally accept options', done => {
const options = {a: 'b'};
const expectedOptions = Object.assign({location: undefined}, options);
const expectedOptions = Object.assign(
{location: undefined, 'formatOptions.useInt64Timestamp': true},
options
);

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand All @@ -252,7 +255,10 @@ describe('BigQuery/Job', () => {
const job = new Job(BIGQUERY, JOB_ID, {location: LOCATION});

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, {location: LOCATION});
assert.deepStrictEqual(reqOpts.qs, {
location: LOCATION,
'formatOptions.useInt64Timestamp': true,
});
done();
};

Expand All @@ -261,7 +267,11 @@ describe('BigQuery/Job', () => {

it('should delete any cached jobs', done => {
const options = {job: {}, a: 'b'};
const expectedOptions = {location: undefined, a: 'b'};
const expectedOptions = {
location: undefined,
a: 'b',
'formatOptions.useInt64Timestamp': true,
};

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand Down Expand Up @@ -340,7 +350,10 @@ describe('BigQuery/Job', () => {
const mergedRows: Array<{}> = [];

const options = {wrapIntegers: true};
const expectedOptions = Object.assign({location: undefined});
const expectedOptions = Object.assign({
location: undefined,
'formatOptions.useInt64Timestamp': true,
});

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand Down Expand Up @@ -368,7 +381,10 @@ describe('BigQuery/Job', () => {
const mergedRows: Array<{}> = [];

const options = {parseJSON: true};
const expectedOptions = Object.assign({location: undefined});
const expectedOptions = Object.assign({
location: undefined,
'formatOptions.useInt64Timestamp': true,
});

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand Down
19 changes: 15 additions & 4 deletions test/table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1979,7 +1979,10 @@ describe('BigQuery/Table', () => {

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
assert.strictEqual(reqOpts.uri, '/data');
assert.strictEqual(reqOpts.qs, options);
assert.deepStrictEqual(reqOpts.qs, {
...options,
'formatOptions.useInt64Timestamp': true,
});
callback(null, {});
};

Expand Down Expand Up @@ -2129,14 +2132,18 @@ describe('BigQuery/Table', () => {
table.metadata = {schema: {}};

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
callback(null, {pageToken});
callback(null, {
'formatOptions.useInt64Timestamp': true,
pageToken,
});
};

table.getRows(options, (err: Error, rows: {}, nextQuery: {}) => {
assert.ifError(err);
assert.deepStrictEqual(nextQuery, {
a: 'b',
c: 'd',
'formatOptions.useInt64Timestamp': true,
pageToken,
});
// Original object isn't affected.
Expand Down Expand Up @@ -2257,7 +2264,9 @@ describe('BigQuery/Table', () => {
const merged = [{name: 'stephen'}];

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
assert.deepStrictEqual(reqOpts.qs, {});
assert.deepStrictEqual(reqOpts.qs, {
'formatOptions.useInt64Timestamp': true,
});
callback(null, {});
};

Expand All @@ -2279,7 +2288,9 @@ describe('BigQuery/Table', () => {
const merged = [{name: 'stephen'}];

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
assert.deepStrictEqual(reqOpts.qs, {});
assert.deepStrictEqual(reqOpts.qs, {
'formatOptions.useInt64Timestamp': true,
});
callback(null, {});
};

Expand Down