Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: switch timestamp representation to int64 usec #1332

Merged
merged 6 commits into from
Feb 21, 2024
Merged

feat: switch timestamp representation to int64 usec #1332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
034d969
feat: switch timestamp representation to int64 usec
alvarowolfx Feb 7, 2024
32ff432
Merge branch 'main' into feat-lossless-timestamp
alvarowolfx Feb 12, 2024
c543b26
fix: properly handle number input
alvarowolfx Feb 16, 2024
fe67d57
fix: lint issues
alvarowolfx Feb 16, 2024
060bc78
Merge branch 'main' into feat-lossless-timestamp
alvarowolfx Feb 19, 2024
5bbc6d9
fix: handle negative unix timestamp values
alvarowolfx Feb 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 5 additions & 16 deletions src/bigquery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,17 +319,17 @@
location?: string;
private _universeDomain: string;

createQueryStream(options?: Query | string): ResourceStream<RowMetadata> {

Check warning on line 322 in src/bigquery.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<RowMetadata>({}, () => {});
}

getDatasetsStream(options?: GetDatasetsOptions): ResourceStream<Dataset> {

Check warning on line 327 in src/bigquery.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<Dataset>({}, () => {});
}

getJobsStream(options?: GetJobsOptions): ResourceStream<Job> {

Check warning on line 332 in src/bigquery.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<Job>({}, () => {});
}
Expand Down Expand Up @@ -2217,15 +2217,13 @@
if (/^\d{4}-\d{1,2}-\d{1,2}/.test(value)) {
pd = new PreciseDate(value);
} else {
const floatValue = Number.parseFloat(value);
if (!Number.isNaN(floatValue)) {
pd = this.fromFloatValue_(floatValue);
} else {
pd = new PreciseDate(value);
}
pd = new PreciseDate(BigInt(value) * BigInt(1000));
}
} else if (value) {
pd = new PreciseDate(BigInt(value) * BigInt(1000));
} else {
pd = this.fromFloatValue_(value);
// Nan or 0 - invalid dates
pd = new PreciseDate(value);
alvarowolfx marked this conversation as resolved.
Show resolved Hide resolved
}
// to keep backward compatibility, only converts with microsecond
// precision if needed.
Expand All @@ -2235,15 +2233,6 @@
this.value = new Date(pd.getTime()).toJSON();
}
}

fromFloatValue_(value: number): PreciseDate {
const secs = Math.trunc(value);
// Timestamps in BigQuery have microsecond precision, so we must
// return a round number of microseconds.
const micros = Math.trunc((value - secs) * 1e6 + 0.5);
const pd = new PreciseDate([secs, micros * 1000]);
return pd;
}
}

/**
Expand Down
2 changes: 1 addition & 1 deletion src/job.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@
location?: string;
projectId?: string;
getQueryResultsStream(
options?: QueryResultsOptions

Check warning on line 130 in src/job.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
): ResourceStream<RowMetadata> {
// placeholder body, overwritten in constructor
return new ResourceStream<RowMetadata>({}, () => {});
Expand Down Expand Up @@ -529,10 +529,10 @@
typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback =
typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;

const qs = extend(
{
location: this.location,
'formatOptions.useInt64Timestamp': true,
},
options
);
Expand Down
11 changes: 9 additions & 2 deletions src/table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@
bigQuery: BigQuery;
location?: string;
rowQueue?: RowQueue;
createReadStream(options?: GetRowsOptions): ResourceStream<RowMetadata> {

Check warning on line 223 in src/table.ts

View workflow job for this annotation

GitHub Actions / lint 

'options' is defined but never used
// placeholder body, overwritten in constructor
return new ResourceStream<RowMetadata>({}, () => {});
}
Expand Down Expand Up @@ -1848,10 +1848,17 @@
callback!(null, rows, nextQuery, resp);
};

const qs = extend(
{
'formatOptions.useInt64Timestamp': true,
},
options
);

this.request(
{
uri: '/data',
qs: options,
qs,
},
(err, resp) => {
if (err) {
Expand All @@ -1860,7 +1867,7 @@
}
let nextQuery: GetRowsOptions | null = null;
if (resp.pageToken) {
nextQuery = Object.assign({}, options, {
nextQuery = Object.assign({}, qs, {
pageToken: resp.pageToken,
});
}
Expand Down
22 changes: 22 additions & 0 deletions test/bigquery.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -850,8 +850,10 @@ describe('BigQuery', () => {
describe('timestamp', () => {
const INPUT_STRING = '2016-12-06T12:00:00.000Z';
const INPUT_STRING_MICROS = '2016-12-06T12:00:00.123456Z';
const INPUT_STRING_NEGATIVE = '1969-12-25T00:00:00.000Z';
const INPUT_DATE = new Date(INPUT_STRING);
const INPUT_PRECISE_DATE = new PreciseDate(INPUT_STRING_MICROS);
const INPUT_PRECISE_NEGATIVE_DATE = new PreciseDate(INPUT_STRING_NEGATIVE);
const EXPECTED_VALUE = INPUT_DATE.toJSON();
const EXPECTED_VALUE_MICROS = INPUT_PRECISE_DATE.toISOString();

Expand Down Expand Up @@ -881,6 +883,26 @@ describe('BigQuery', () => {
assert.strictEqual(timestamp.value, EXPECTED_VALUE);
});

it('should accept a number in microseconds', () => {
let ms = INPUT_PRECISE_DATE.valueOf(); // milliseconds
let us = ms * 1000 + INPUT_PRECISE_DATE.getMicroseconds(); // microseconds
let timestamp = bq.timestamp(us);
assert.strictEqual(timestamp.value, EXPECTED_VALUE_MICROS);

let usStr = `${us}`;
timestamp = bq.timestamp(usStr);
assert.strictEqual(timestamp.value, EXPECTED_VALUE_MICROS);

ms = INPUT_PRECISE_NEGATIVE_DATE.valueOf();
us = ms * 1000;
timestamp = bq.timestamp(us);
assert.strictEqual(timestamp.value, INPUT_STRING_NEGATIVE);

usStr = `${us}`;
timestamp = bq.timestamp(usStr);
assert.strictEqual(timestamp.value, INPUT_STRING_NEGATIVE);
});

it('should accept a string with microseconds', () => {
const timestamp = bq.timestamp(INPUT_STRING_MICROS);
assert.strictEqual(timestamp.value, EXPECTED_VALUE_MICROS);
Expand Down
26 changes: 21 additions & 5 deletions test/job.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,10 @@ describe('BigQuery/Job', () => {

it('should optionally accept options', done => {
const options = {a: 'b'};
const expectedOptions = Object.assign({location: undefined}, options);
const expectedOptions = Object.assign(
{location: undefined, 'formatOptions.useInt64Timestamp': true},
options
);

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand All @@ -252,7 +255,10 @@ describe('BigQuery/Job', () => {
const job = new Job(BIGQUERY, JOB_ID, {location: LOCATION});

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, {location: LOCATION});
assert.deepStrictEqual(reqOpts.qs, {
location: LOCATION,
'formatOptions.useInt64Timestamp': true,
});
done();
};

Expand All @@ -261,7 +267,11 @@ describe('BigQuery/Job', () => {

it('should delete any cached jobs', done => {
const options = {job: {}, a: 'b'};
const expectedOptions = {location: undefined, a: 'b'};
const expectedOptions = {
location: undefined,
a: 'b',
'formatOptions.useInt64Timestamp': true,
};

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand Down Expand Up @@ -340,7 +350,10 @@ describe('BigQuery/Job', () => {
const mergedRows: Array<{}> = [];

const options = {wrapIntegers: true};
const expectedOptions = Object.assign({location: undefined});
const expectedOptions = Object.assign({
location: undefined,
'formatOptions.useInt64Timestamp': true,
});

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand Down Expand Up @@ -368,7 +381,10 @@ describe('BigQuery/Job', () => {
const mergedRows: Array<{}> = [];

const options = {parseJSON: true};
const expectedOptions = Object.assign({location: undefined});
const expectedOptions = Object.assign({
location: undefined,
'formatOptions.useInt64Timestamp': true,
});

BIGQUERY.request = (reqOpts: DecorateRequestOptions) => {
assert.deepStrictEqual(reqOpts.qs, expectedOptions);
Expand Down
19 changes: 15 additions & 4 deletions test/table.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1979,7 +1979,10 @@ describe('BigQuery/Table', () => {

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
assert.strictEqual(reqOpts.uri, '/data');
assert.strictEqual(reqOpts.qs, options);
assert.deepStrictEqual(reqOpts.qs, {
...options,
'formatOptions.useInt64Timestamp': true,
});
callback(null, {});
};

Expand Down Expand Up @@ -2129,14 +2132,18 @@ describe('BigQuery/Table', () => {
table.metadata = {schema: {}};

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
callback(null, {pageToken});
callback(null, {
'formatOptions.useInt64Timestamp': true,
pageToken,
});
};

table.getRows(options, (err: Error, rows: {}, nextQuery: {}) => {
assert.ifError(err);
assert.deepStrictEqual(nextQuery, {
a: 'b',
c: 'd',
'formatOptions.useInt64Timestamp': true,
pageToken,
});
// Original object isn't affected.
Expand Down Expand Up @@ -2257,7 +2264,9 @@ describe('BigQuery/Table', () => {
const merged = [{name: 'stephen'}];

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
assert.deepStrictEqual(reqOpts.qs, {});
assert.deepStrictEqual(reqOpts.qs, {
'formatOptions.useInt64Timestamp': true,
});
callback(null, {});
};

Expand All @@ -2279,7 +2288,9 @@ describe('BigQuery/Table', () => {
const merged = [{name: 'stephen'}];

table.request = (reqOpts: DecorateRequestOptions, callback: Function) => {
assert.deepStrictEqual(reqOpts.qs, {});
assert.deepStrictEqual(reqOpts.qs, {
'formatOptions.useInt64Timestamp': true,
});
callback(null, {});
};

Expand Down