Skip to content

Commit

Permalink
faster timestamp transform
Browse files Browse the repository at this point in the history
  • Loading branch information
vitalyisaev2 committed Dec 20, 2024
1 parent b1b683c commit dfba480
Show file tree
Hide file tree
Showing 3 changed files with 223 additions and 4 deletions.
6 changes: 2 additions & 4 deletions app/server/conversion/converters_default.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import (
"fmt"
"time"

"github.com/phuslu/fasttime"

"github.com/ydb-platform/fq-connector-go/common"
)

Expand Down Expand Up @@ -139,11 +137,11 @@ func (timestampToStringConverterUTC) Convert(in *time.Time) (string, error) {
// ClickHouse - 1 nanosecond (10^-9 s)
// Oracle - 1 nanosecond (10^-9 s)
// Trailing zeros are omitted
return fasttime.Strftime("%Y-%m-%dT%H:%M:%S.%N%:z", in.UTC()), nil
return in.UTC().Format("2006-01-02T15:04:05.999999999Z"), nil
}

type timestampToStringConverterNaive struct{}

func (timestampToStringConverterNaive) Convert(in *time.Time) (string, error) {
return fasttime.Strftime("%Y-%m-%dT%H:%M:%S.%N%:z", *in), nil
return in.Format("2006-01-02T15:04:05.999999999"), nil
}
66 changes: 66 additions & 0 deletions app/server/conversion/converters_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,69 @@ func FuzzDateToStringConverter(f *testing.F) {
require.Equal(t, expectedOut, actualOut)
})
}

func TestTimestampToStringConverter(t *testing.T) {
testCases := []time.Time{
time.Date(math.MaxInt, math.MaxInt, math.MaxInt, math.MaxInt, math.MaxInt, math.MaxInt, math.MaxInt, time.UTC),
time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
time.Date(math.MinInt, math.MinInt, math.MinInt, math.MinInt, math.MinInt, math.MinInt, math.MinInt, time.UTC),
time.Date(1950, 5, 27, 0, 0, 0, 0, time.UTC),
time.Date(1950, 5, 27, 0, 0, 0, 1, time.UTC),
time.Date(1950, 5, 27, 1, 2, 3, 12345678, time.UTC),
time.Date(1950, 5, 27, 13, 14, 15, 123456789, time.UTC),
time.Date(-1, 1, 1, 0, 0, 0, 0, time.UTC),
time.Date(100500, 5, 20, -12, -55, -8, 0, time.UTC),
time.Date(1988, 11, 20, 12, 55, 8, 0, time.UTC),
time.Date(100, 2, 3, 4, 5, 6, 7, time.UTC),
time.Date(10, 2, 3, 4, 5, 6, 7, time.UTC),
time.Date(1, 2, 3, 4, 5, 6, 7, time.UTC),
time.Date(-100500, -10, -35, -8, -2000, -300000, -50404040, time.UTC),
}

const format = time.RFC3339Nano

var (
converterUnsafe timestampToStringConverterUTCUnsafe
converterDefault timestampToStringConverterUTC
)

for _, tc := range testCases {
tc := tc
t.Run(tc.Format(format), func(t *testing.T) {
// Check equivalence of results produced by default and unsafe converters
expectedOut, err := converterDefault.Convert(&tc)
require.NoError(t, err)
actualOut, err := converterUnsafe.Convert(&tc)
require.NoError(t, err)
require.Equal(t, expectedOut, actualOut)
})
}
}

func BenchmarkTimestampToStringConverter(b *testing.B) {
t := time.Now()

b.Run("Default", func(b *testing.B) {
var converter timestampToStringConverterUTC

for i := 0; i < b.N; i++ {
out, err := converter.Convert(&t)
if err != nil {
b.Fatal(err)
}
_ = out
}
})

b.Run("Unsafe", func(b *testing.B) {
var converter timestampToStringConverterUTCUnsafe

for i := 0; i < b.N; i++ {
out, err := converter.Convert(&t)
if err != nil {
b.Fatal(err)
}
_ = out
}
})
}
155 changes: 155 additions & 0 deletions app/server/conversion/converters_unsafe.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,158 @@ func (dateToStringConverterUnsafe) Convert(in *time.Time) (string, error) {

return unsafe.String(p, len(buf)), nil
}

type timestampToStringConverterUTCUnsafe struct{}

func (timestampToStringConverterUTCUnsafe) Convert(in *time.Time) (string, error) {
buf := make([]byte, 0, 32)
year, month, day := in.Date()

// year

if year < 0 {
buf = append(buf, byte('-'))
}

absYear := absInt(year)

switch {
case absYear < 10:
buf = append(buf, []byte("000")...)
case absYear < 100:
buf = append(buf, []byte("00")...)
case absYear < 1000:
buf = append(buf, byte('0'))
}

buf, _ = formatBits(buf, uint64(absYear), 10, false, true)

// month

buf = append(buf, byte('-'))
if month < 10 {
buf = append(buf, byte('0'))
}

buf, _ = formatBits(buf, uint64(month), 10, false, true)

// day

buf = append(buf, byte('-'))
if day < 10 {
buf = append(buf, byte('0'))
}

buf, _ = formatBits(buf, uint64(day), 10, false, true)

// T
buf = append(buf, byte('T'))

hour, minutes, seconds := in.Clock()

// hours

if hour < 10 {
buf = append(buf, byte('0'))
}

buf, _ = formatBits(buf, uint64(hour), 10, false, true)

buf = append(buf, byte(':'))

// minutes

if minutes < 10 {
buf = append(buf, byte('0'))
}

buf, _ = formatBits(buf, uint64(minutes), 10, false, true)

buf = append(buf, byte(':'))

// seconds

if seconds < 10 {
buf = append(buf, byte('0'))
}

buf, _ = formatBits(buf, uint64(seconds), 10, false, true)

// nanoseconds

nanoseconds := in.Nanosecond()
if nanoseconds > 0 {
buf = append(buf, byte('.'))

buf = formatNanoseconds(buf, nanoseconds)
}

buf = append(buf, byte('Z'))

p := unsafe.SliceData(buf)

return unsafe.String(p, len(buf)), nil
}

const tab = "00010203040506070809" +
"10111213141516171819" +
"20212223242526272829" +
"30313233343536373839" +
"40414243444546474849" +
"50515253545556575859" +
"60616263646566676869" +
"70717273747576777879" +
"80818283848586878889" +
"90919293949596979899"

func formatNanoseconds(buf []byte, ns int) []byte {
// fast transformation of nanoseconds
var tmp [9]byte
b := ns % 100 * 2

Check failure on line 178 in app/server/conversion/converters_unsafe.go

View workflow job for this annotation

GitHub Actions / lint

assignments should only be cuddled with other assignments (wsl)
tmp[8] = tab[b+1]
tmp[7] = tab[b]
ns /= 100
b = ns % 100 * 2
tmp[6] = tab[b+1]
tmp[5] = tab[b]
ns /= 100
b = ns % 100 * 2
tmp[4] = tab[b+1]
tmp[3] = tab[b]
ns /= 100
b = ns % 100 * 2
tmp[2] = tab[b+1]
tmp[1] = tab[b]
tmp[0] = byte(ns/100) + '0'

// check for trailing zeroes
i := 8
for ; i >= 0; i-- {
if tmp[i] != '0' {
break
}
}

buf = append(buf, tmp[:i+1]...)

return buf
}

func digitsInNumber(n int) int {

Check failure on line 208 in app/server/conversion/converters_unsafe.go

View workflow job for this annotation

GitHub Actions / lint

func `digitsInNumber` is unused (unused)
if n < 0 {
n = -n
}

Check warning on line 211 in app/server/conversion/converters_unsafe.go

View check run for this annotation

Codecov / codecov/patch

app/server/conversion/converters_unsafe.go#L208-L211

Added lines #L208 - L211 were not covered by tests

if n == 0 {
return 1
}

Check warning on line 215 in app/server/conversion/converters_unsafe.go

View check run for this annotation

Codecov / codecov/patch

app/server/conversion/converters_unsafe.go#L213-L215

Added lines #L213 - L215 were not covered by tests

digits := 0

for n > 0 {
digits++
n /= 10

Check failure on line 221 in app/server/conversion/converters_unsafe.go

View workflow job for this annotation

GitHub Actions / lint

assignments should only be cuddled with other assignments (wsl)
}

Check warning on line 222 in app/server/conversion/converters_unsafe.go

View check run for this annotation

Codecov / codecov/patch

app/server/conversion/converters_unsafe.go#L217-L222

Added lines #L217 - L222 were not covered by tests

return digits

Check warning on line 224 in app/server/conversion/converters_unsafe.go

View check run for this annotation

Codecov / codecov/patch

app/server/conversion/converters_unsafe.go#L224

Added line #L224 was not covered by tests
}

0 comments on commit dfba480

Please sign in to comment.