Skip to content

Commit 04b284c

Browse files
nmemondnmemond
committed
Add data types test for bulk migration from MySQL to a Postgres dialect Spanner DB
1 parent a074388 commit 04b284c

File tree

3 files changed

+657
-32
lines changed

3 files changed

+657
-32
lines changed

‎v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java‎

Lines changed: 171 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
import org.apache.beam.it.gcp.spanner.SpannerResourceManager;
3434
import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts;
3535
import org.apache.beam.it.jdbc.MySQLResourceManager;
36-
import org.junit.After;
36+
import org.junit.AfterClass;
3737
import org.junit.Before;
3838
import org.junit.Test;
3939
import org.junit.experimental.categories.Category;
@@ -53,31 +53,44 @@ public class MySQLDataTypesIT extends SourceDbToSpannerITBase {
5353
private static final Logger LOG = LoggerFactory.getLogger(MySQLDataTypesIT.class);
5454
private static PipelineLauncher.LaunchInfo jobInfo;
5555

56+
private static boolean initialized = false;
5657
public static MySQLResourceManager mySQLResourceManager;
5758
public static SpannerResourceManager spannerResourceManager;
59+
public static SpannerResourceManager pgDialectSpannerResourceManager;
5860

5961
private static final String MYSQL_DUMP_FILE_RESOURCE = "DataTypesIT/mysql-data-types.sql";
6062

6163
private static final String SPANNER_DDL_RESOURCE = "DataTypesIT/mysql-spanner-schema.sql";
64+
private static final String PG_DIALECT_SPANNER_DDL_RESOURCE =
65+
"DataTypesIT/mysql-pg-dialect-spanner-schema.sql";
6266

6367
/**
6468
* Setup resource managers and Launch dataflow job once during the execution of this test class. \
6569
*/
6670
@Before
67-
public void setUp() {
68-
mySQLResourceManager = setUpMySQLResourceManager();
69-
spannerResourceManager = setUpSpannerResourceManager();
71+
public void setUp() throws Exception {
72+
synchronized (MySQLSourceDBToSpannerWideRowMaxSizeStringIT.class) {
73+
if (!initialized) {
74+
mySQLResourceManager = setUpMySQLResourceManager();
75+
spannerResourceManager = setUpSpannerResourceManager();
76+
pgDialectSpannerResourceManager = setUpPGDialectSpannerResourceManager();
77+
78+
loadSQLFileResource(mySQLResourceManager, MYSQL_DUMP_FILE_RESOURCE);
79+
80+
initialized = true;
81+
}
82+
}
7083
}
7184

7285
/** Cleanup dataflow job and all the resources and resource managers. */
73-
@After
74-
public void cleanUp() {
75-
ResourceManagerUtils.cleanResources(spannerResourceManager, mySQLResourceManager);
86+
@AfterClass
87+
public static void cleanUp() {
88+
ResourceManagerUtils.cleanResources(
89+
spannerResourceManager, pgDialectSpannerResourceManager, mySQLResourceManager);
7690
}
7791

7892
@Test
7993
public void allTypesTest() throws Exception {
80-
loadSQLFileResource(mySQLResourceManager, MYSQL_DUMP_FILE_RESOURCE);
8194
createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE);
8295
jobInfo =
8396
launchDataflowJob(
@@ -94,13 +107,39 @@ public void allTypesTest() throws Exception {
94107

95108
// Validate supported data types.
96109
Map<String, List<Map<String, Object>>> expectedData = getExpectedData();
110+
validateResult(spannerResourceManager, expectedData);
111+
}
112+
113+
@Test
114+
public void allTypesTestPGDialect() throws Exception {
115+
createSpannerDDL(pgDialectSpannerResourceManager, PG_DIALECT_SPANNER_DDL_RESOURCE);
116+
jobInfo =
117+
launchDataflowJob(
118+
getClass().getSimpleName(),
119+
null,
120+
null,
121+
mySQLResourceManager,
122+
pgDialectSpannerResourceManager,
123+
null,
124+
null);
125+
PipelineOperator.Result result =
126+
pipelineOperator().waitUntilDone(createConfig(jobInfo, Duration.ofMinutes(35L)));
127+
assertThatResult(result).isLaunchFinished();
128+
129+
// Validate supported data types.
130+
Map<String, List<Map<String, Object>>> expectedData = getExpectedDataPGDialect();
131+
validateResult(pgDialectSpannerResourceManager, expectedData);
132+
}
133+
134+
private void validateResult(
135+
SpannerResourceManager resourceManager, Map<String, List<Map<String, Object>>> expectedData) {
97136
for (Map.Entry<String, List<Map<String, Object>>> entry : expectedData.entrySet()) {
98137
String type = entry.getKey();
99138
String tableName = String.format("%s_table", type);
100139
String colName = String.format("%s_col", type);
101140
LOG.info("Asserting type: {}", type);
102141

103-
List<Struct> rows = spannerResourceManager.readTableRecords(tableName, "id", colName);
142+
List<Struct> rows = resourceManager.readTableRecords(tableName, "id", colName);
104143
for (Struct row : rows) {
105144
// Limit logs printed for very large strings.
106145
String rowString = row.toString();
@@ -127,7 +166,7 @@ public void allTypesTest() throws Exception {
127166

128167
for (String table : unsupportedTypeTables) {
129168
// Unsupported rows should still be migrated. Each source table has 1 row.
130-
assertThat(spannerResourceManager.getRowCount(table)).isEqualTo(1L);
169+
assertThat(resourceManager.getRowCount(table)).isEqualTo(1L);
131170
}
132171
}
133172

@@ -155,7 +194,8 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
155194
createRows("bigint", "40", "9223372036854775807", "-9223372036854775808", "NULL"));
156195
expectedData.put(
157196
"bigint_to_string",
158-
createRows("bigint_to_string", "40", "9223372036854775807", "-9223372036854775808", "NULL"));
197+
createRows(
198+
"bigint_to_string", "40", "9223372036854775807", "-9223372036854775808", "NULL"));
159199
expectedData.put(
160200
"bigint_unsigned",
161201
createRows("bigint_unsigned", "42", "0", "18446744073709551615", "NULL"));
@@ -164,15 +204,21 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
164204
createRows("binary", "eDU4MD" + repeatString("A", 334), repeatString("/", 340), "NULL"));
165205
expectedData.put(
166206
"binary_to_string",
167-
createRows("binary_to_string", "783538303000000000000000000000000...", "fffffffffffffffffffffffffffffffff...", "NULL"));
207+
createRows(
208+
"binary_to_string",
209+
"783538303000000000000000000000000...",
210+
"fffffffffffffffffffffffffffffffff...",
211+
"NULL"));
168212
expectedData.put("bit", createRows("bit", "f/////////8=", "NULL"));
169213
expectedData.put("bit8", createRows("bit8", "0", "255", "NULL"));
170214
expectedData.put("bit1", createRows("bit1", "false", "true", "NULL"));
171215
expectedData.put("bit_to_bool", createRows("bit_to_bool", "false", "true", "NULL"));
172216
expectedData.put("bit_to_string", createRows("bit_to_string", "0", "1", "NULL"));
173217
expectedData.put("bit_to_int64", createRows("bit_to_int64", "9223372036854775807", "NULL"));
174218
expectedData.put("blob", createRows("blob", "eDU4MDA=", repeatString("/", 87380), "NULL"));
175-
expectedData.put("blob_to_string", createRows("blob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
219+
expectedData.put(
220+
"blob_to_string",
221+
createRows("blob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
176222
expectedData.put("bool", createRows("bool", "false", "true", "NULL"));
177223
expectedData.put("bool_to_string", createRows("bool_to_string", "0", "1", "NULL"));
178224
expectedData.put("boolean", createRows("boolean", "false", "true", "NULL"));
@@ -181,9 +227,11 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
181227
expectedData.put(
182228
"char", createRows("char", "a", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...", "NULL"));
183229
expectedData.put("date", createRows("date", "2012-09-17", "1000-01-01", "9999-12-31", "NULL"));
184-
// date_to_string is commented out to avoid failing the test case; returned data has format "YYYY-MM-DDTHH:mm:SSZ"
230+
// date_to_string is commented out to avoid failing the test case; returned data has format
231+
// "YYYY-MM-DDTHH:mm:SSZ"
185232
// which is unexpected even if it's not necessarily incorrect
186-
// expectedData.put("date_to_string", createRows("date_to_string", "2012-09-17", "1000-01-01", "9999-12-31", "NULL"));
233+
// expectedData.put("date_to_string", createRows("date_to_string", "2012-09-17", "1000-01-01",
234+
// "9999-12-31", "NULL"));
187235
expectedData.put(
188236
"datetime",
189237
createRows(
@@ -234,37 +282,63 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
234282
"NULL"));
235283
expectedData.put(
236284
"double_precision_to_float64",
237-
createRows("double_precision_to_float64", "52.67", "1.7976931348623157E308", "-1.7976931348623157E308", "NULL"));
285+
createRows(
286+
"double_precision_to_float64",
287+
"52.67",
288+
"1.7976931348623157E308",
289+
"-1.7976931348623157E308",
290+
"NULL"));
238291
expectedData.put(
239292
"double_precision_to_string",
240-
createRows("double_precision_to_string", "52.67", "1.7976931348623157E308", "-1.7976931348623157E308", "NULL"));
293+
createRows(
294+
"double_precision_to_string",
295+
"52.67",
296+
"1.7976931348623157E308",
297+
"-1.7976931348623157E308",
298+
"NULL"));
241299
expectedData.put(
242300
"double",
243301
createRows("double", "52.67", "1.7976931348623157E308", "-1.7976931348623157E308", "NULL"));
244302
expectedData.put(
245303
"double_to_string",
246-
createRows("double_to_string", "52.67", "1.7976931348623157E308", "-1.7976931348623157E308", "NULL"));
304+
createRows(
305+
"double_to_string",
306+
"52.67",
307+
"1.7976931348623157E308",
308+
"-1.7976931348623157E308",
309+
"NULL"));
247310
expectedData.put("enum", createRows("enum", "1", "NULL"));
248311
expectedData.put("float", createRows("float", "45.56", "3.4E38", "-3.4E38", "NULL"));
249-
expectedData.put("float_to_float32", createRows("float_to_float32", "45.56", "3.4E38", "-3.4E38", "NULL"));
250-
expectedData.put("float_to_string", createRows("float_to_string", "45.56", "3.4E38", "-3.4E38", "NULL"));
312+
expectedData.put(
313+
"float_to_float32", createRows("float_to_float32", "45.56", "3.4E38", "-3.4E38", "NULL"));
314+
expectedData.put(
315+
"float_to_string", createRows("float_to_string", "45.56", "3.4E38", "-3.4E38", "NULL"));
251316
expectedData.put("int", createRows("int", "30", "2147483647", "-2147483648", "NULL"));
252-
expectedData.put("int_to_string", createRows("int_to_string", "30", "2147483647", "-2147483648", "NULL"));
253-
expectedData.put("integer_to_int64", createRows("integer_to_int64", "30", "2147483647", "-2147483648", "NULL"));
254-
expectedData.put("integer_to_string", createRows("integer_to_string", "30", "2147483647", "-2147483648", "NULL"));
317+
expectedData.put(
318+
"int_to_string", createRows("int_to_string", "30", "2147483647", "-2147483648", "NULL"));
319+
expectedData.put(
320+
"integer_to_int64",
321+
createRows("integer_to_int64", "30", "2147483647", "-2147483648", "NULL"));
322+
expectedData.put(
323+
"integer_to_string",
324+
createRows("integer_to_string", "30", "2147483647", "-2147483648", "NULL"));
255325
expectedData.put("test_json", createRows("test_json", "{\"k1\":\"v1\"}", "NULL"));
256326
expectedData.put("json_to_string", createRows("json_to_string", "{\"k1\": \"v1\"}", "NULL"));
257327
expectedData.put(
258328
"longblob", createRows("longblob", "eDU4MDA=", repeatString("/", 87380), "NULL"));
259329
expectedData.put(
260-
"longblob_to_string", createRows("longblob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
330+
"longblob_to_string",
331+
createRows(
332+
"longblob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
261333
expectedData.put(
262334
"longtext",
263335
createRows("longtext", "longtext", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...", "NULL"));
264336
expectedData.put(
265337
"mediumblob", createRows("mediumblob", "eDU4MDA=", repeatString("/", 87380), "NULL"));
266338
expectedData.put(
267-
"mediumblob_to_string", createRows("mediumblob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
339+
"mediumblob_to_string",
340+
createRows(
341+
"mediumblob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
268342
expectedData.put("mediumint", createRows("mediumint", "20", "NULL"));
269343
expectedData.put("mediumint_to_string", createRows("mediumint_to_string", "20", "NULL"));
270344
expectedData.put(
@@ -290,14 +364,26 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
290364
"NULL"));
291365
expectedData.put(
292366
"real_to_float64",
293-
createRows("real_to_float64", "52.67", "1.7976931348623157E308", "-1.7976931348623157E308", "NULL"));
367+
createRows(
368+
"real_to_float64",
369+
"52.67",
370+
"1.7976931348623157E308",
371+
"-1.7976931348623157E308",
372+
"NULL"));
294373
expectedData.put(
295374
"real_to_string",
296-
createRows("real_to_string", "52.67", "1.7976931348623157E308", "-1.7976931348623157E308", "NULL"));
297-
// set_to_array is commented out to avoid failing the test case; data does not get migrated at all
375+
createRows(
376+
"real_to_string",
377+
"52.67",
378+
"1.7976931348623157E308",
379+
"-1.7976931348623157E308",
380+
"NULL"));
381+
// set_to_array is commented out to avoid failing the test case; data does not get migrated at
382+
// all
298383
// expectedData.put("set_to_array", createRows("set_to_array", "v1,v2", "NULL"));
299384
expectedData.put("smallint", createRows("smallint", "15", "32767", "-32768", "NULL"));
300-
expectedData.put("smallint_to_string", createRows("smallint_to_string", "15", "32767", "-32768", "NULL"));
385+
expectedData.put(
386+
"smallint_to_string", createRows("smallint_to_string", "15", "32767", "-32768", "NULL"));
301387
expectedData.put(
302388
"smallint_unsigned", createRows("smallint_unsigned", "42", "0", "65535", "NULL"));
303389
expectedData.put("text", createRows("text", "xyz", repeatString("a", 33) + "...", "NULL"));
@@ -321,17 +407,22 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
321407
expectedData.put(
322408
"tinyblob", createRows("tinyblob", "eDU4MDA=", repeatString("/", 340), "NULL"));
323409
expectedData.put(
324-
"tinyblob_to_string", createRows("tinyblob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
410+
"tinyblob_to_string",
411+
createRows(
412+
"tinyblob_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
325413
expectedData.put("tinyint", createRows("tinyint", "10", "127", "-128", "NULL"));
326-
expectedData.put("tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", "NULL"));
414+
expectedData.put(
415+
"tinyint_to_string", createRows("tinyint_to_string", "10", "127", "-128", "NULL"));
327416
expectedData.put("tinyint_unsigned", createRows("tinyint_unsigned", "0", "255", "NULL"));
328417
expectedData.put(
329418
"tinytext",
330419
createRows("tinytext", "tinytext", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...", "NULL"));
331420
expectedData.put(
332421
"varbinary", createRows("varbinary", "eDU4MDA=", repeatString("/", 86666) + "8=", "NULL"));
333422
expectedData.put(
334-
"varbinary_to_string", createRows("varbinary_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
423+
"varbinary_to_string",
424+
createRows(
425+
"varbinary_to_string", "7835383030", "fffffffffffffffffffffffffffffffff...", "NULL"));
335426
expectedData.put(
336427
"varchar", createRows("varchar", "abc", repeatString("a", 33) + "...", "NULL"));
337428
expectedData.put("year", createRows("year", "2022", "1901", "2155", "NULL"));
@@ -386,6 +477,54 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
386477
return expectedData;
387478
}
388479

480+
private Map<String, List<Map<String, Object>>> getExpectedDataPGDialect() {
481+
// Expected data for PG dialect is roughly similar to the spanner dialect data, with some minor
482+
// differences. Notably,
483+
// we aren't testing PK data types yet, and some data types like numeric have slightly different
484+
// behaviour.
485+
Map<String, List<Map<String, Object>>> expectedData = getExpectedData();
486+
487+
expectedData.keySet().removeIf(column -> column.endsWith("_pk"));
488+
489+
expectedData.put(
490+
"bigint_unsigned",
491+
createRows(
492+
"bigint_unsigned",
493+
"42.000000000",
494+
"0.000000000",
495+
"18446744073709551615.000000000",
496+
"NULL"));
497+
expectedData.put(
498+
"dec_to_numeric",
499+
createRows(
500+
"dec_to_numeric",
501+
"68.750000000",
502+
"99999999999999999999999.999999999",
503+
"12345678912345678.123456789",
504+
"NULL"));
505+
expectedData.put(
506+
"decimal",
507+
createRows(
508+
"decimal",
509+
"68.750000000",
510+
"99999999999999999999999.999999999",
511+
"12345678912345678.123456789",
512+
"NULL"));
513+
// The data in this table fails to migrate, removing to avoid test failure
514+
expectedData.remove("float_to_float32");
515+
expectedData.put("test_json", createRows("test_json", "{\"k1\": \"v1\"}", "NULL"));
516+
expectedData.put(
517+
"numeric_to_numeric",
518+
createRows(
519+
"numeric_to_numeric",
520+
"68.750000000",
521+
"99999999999999999999999.999999999",
522+
"12345678912345678.123456789",
523+
"NULL"));
524+
525+
return expectedData;
526+
}
527+
389528
private static String repeatString(String str, int count) {
390529
return new String(new char[count]).replace("\0", str);
391530
}

‎v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/SourceDbToSpannerITBase.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline;
1919

20+
import com.google.cloud.spanner.Dialect;
2021
import com.google.cloud.teleport.v2.source.reader.io.jdbc.iowrapper.config.SQLDialect;
2122
import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation;
2223
import com.google.common.io.Resources;
@@ -85,6 +86,12 @@ public SpannerResourceManager setUpSpannerResourceManager() {
8586
.build();
8687
}
8788

89+
public SpannerResourceManager setUpPGDialectSpannerResourceManager() {
90+
return SpannerResourceManager.builder(testName, PROJECT, REGION, Dialect.POSTGRESQL)
91+
.maybeUseStaticInstance()
92+
.build();
93+
}
94+
8895
protected void loadSQLFileResource(JDBCResourceManager jdbcResourceManager, String resourcePath)
8996
throws Exception {
9097
String sql =

0 commit comments

Comments
 (0)