Skip to content
This repository has been archived by the owner on Aug 23, 2021. It is now read-only.

Fix some tests (HSQLDB quirks, case confusion, supplemental procedures). #387

Merged
merged 3 commits into from
Aug 1, 2021
Merged

Fix some tests (HSQLDB quirks, case confusion, supplemental procedures). #387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c454790
Fix TATPLoader.
lmwnshn Jul 29, 2021
7dc89c6
Fix TestSEATSLoader and TestSEATSWorker.
lmwnshn Jul 30, 2021
703e333
Fix TestUserIdGenerator.
lmwnshn Jul 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions src/main/java/com/oltpbenchmark/api/BenchmarkModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,10 +269,10 @@ public final void createDatabase(DatabaseType dbType, Connection conn) {
/**
* Invoke this benchmark's database loader
*/
public final void loadDatabase() {

public final Loader<? extends BenchmarkModule> loadDatabase() {
Loader<? extends BenchmarkModule> loader;
try {
Loader<? extends BenchmarkModule> loader = this.makeLoaderImpl();
loader = this.makeLoaderImpl();
if (loader != null) {


Expand Down Expand Up @@ -303,6 +303,7 @@ public final void loadDatabase() {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Finished loading the %s database", this.getBenchmarkName().toUpperCase()));
}
return loader;
}

public final void clearDatabase() {
Expand Down Expand Up @@ -376,7 +377,7 @@ public final TransactionType initTransactionType(String procName, int id) {
String fullName = pkg.getName() + "." + procName;
Class<? extends Procedure> procClass = (Class<? extends Procedure>) ClassUtil.getClass(fullName);

return new TransactionType(procClass, id);
return new TransactionType(procClass, id, false);
}

public final WorkloadConfiguration getWorkloadConfiguration() {
Expand All @@ -396,7 +397,7 @@ public Map<TransactionType, Procedure> getProcedures() {
for (Class<? extends Procedure> procClass : this.supplementalProcedures) {
TransactionType txn = txns.getType(procClass);
if (txn == null) {
txn = new TransactionType(procClass, procClass.hashCode());
txn = new TransactionType(procClass, procClass.hashCode(), true);
txns.add(txn);
}
}
Expand Down
10 changes: 8 additions & 2 deletions src/main/java/com/oltpbenchmark/api/TransactionType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,16 @@ public static class Invalid extends Procedure {
}

public static final int INVALID_ID = 0;
public static final TransactionType INVALID = new TransactionType(Invalid.class, INVALID_ID);
public static final TransactionType INVALID = new TransactionType(Invalid.class, INVALID_ID, false);

private final Class<? extends Procedure> procClass;
private final int id;
private final boolean supplemental;

protected TransactionType(Class<? extends Procedure> procClass, int id) {
protected TransactionType(Class<? extends Procedure> procClass, int id, boolean supplemental) {
this.procClass = procClass;
this.id = id;
this.supplemental = supplemental;
}

public Class<? extends Procedure> getProcedureClass() {
Expand All @@ -48,6 +50,10 @@ public int getId() {
return this.id;
}

public boolean isSupplemental() {
return this.supplemental;
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand Down
24 changes: 12 additions & 12 deletions src/main/java/com/oltpbenchmark/benchmarks/auctionmark/AuctionMarkLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ protected AbstractTableGenerator getGenerator(String table_name) {
* @param tableName
*/
protected void generateTableData(Connection conn, String tableName) throws SQLException {
LOG.info("*** START {}", tableName);
LOG.debug("*** START {}", tableName);
final AbstractTableGenerator generator = this.generators.get(tableName);


Expand Down Expand Up @@ -256,7 +256,7 @@ protected void generateTableData(Connection conn, String tableName) throws SQLEx
generator.markAsFinished();
synchronized (this) {
this.finished.add(tableName);
LOG.info(String.format("*** FINISH %s - %d tuples - [%d / %d]", tableName, this.tableSizes.get(tableName), this.finished.size(), this.generators.size()));
LOG.debug(String.format("*** FINISH %s - %d tuples - [%d / %d]", tableName, this.tableSizes.get(tableName), this.finished.size(), this.generators.size()));
if (LOG.isDebugEnabled()) {
LOG.debug("Remaining Tables: {}", CollectionUtils.subtract(this.generators.keySet(), this.finished));
}
Expand Down Expand Up @@ -311,16 +311,16 @@ public AbstractTableGenerator(String tableName, String... dependencies) {
this.batchSize = workConf.getBatchSize();


boolean fixed_size = AuctionMarkConstants.FIXED_TABLES.contains(catalog_tbl.getUppercaseName());
boolean dynamic_size = AuctionMarkConstants.DYNAMIC_TABLES.contains(catalog_tbl.getUppercaseName());
boolean data_file = AuctionMarkConstants.DATAFILE_TABLES.contains(catalog_tbl.getUppercaseName());
boolean fixed_size = AuctionMarkConstants.FIXED_TABLES.contains(catalog_tbl.getName().toLowerCase());
boolean dynamic_size = AuctionMarkConstants.DYNAMIC_TABLES.contains(catalog_tbl.getName().toLowerCase());
boolean data_file = AuctionMarkConstants.DATAFILE_TABLES.contains(catalog_tbl.getName().toLowerCase());

// Add the dependencies so that we know what we need to block on
CollectionUtil.addAll(this.dependencyTables, dependencies);

// Initialize dynamic parameters for tables that are not loaded from data files
if (!data_file && !dynamic_size && !tableName.equalsIgnoreCase(AuctionMarkConstants.TABLENAME_ITEM)) {
String field_name = "TABLESIZE_" + StringUtils.upperCase(catalog_tbl.getUppercaseName());
String field_name = "TABLESIZE_" + catalog_tbl.getName().toUpperCase();
try {

Field field_handle = AuctionMarkConstants.class.getField(field_name);
Expand All @@ -337,17 +337,17 @@ public AbstractTableGenerator(String tableName, String... dependencies) {
}

for (Column catalog_col : this.catalog_tbl.getColumns()) {
if (random_str_regex.matcher(catalog_col.getName()).matches()) {
if (random_str_regex.matcher(catalog_col.getName().toUpperCase()).matches()) {

this.random_str_cols.add(catalog_col);
if (LOG.isTraceEnabled()) {
LOG.trace("Random String Column: {}", catalog_col.getName());
LOG.trace("Random String Column: {}", catalog_col.getName().toLowerCase());
}
} else if (random_int_regex.matcher(catalog_col.getName()).matches()) {
} else if (random_int_regex.matcher(catalog_col.getName().toUpperCase()).matches()) {

this.random_int_cols.add(catalog_col);
if (LOG.isTraceEnabled()) {
LOG.trace("Random Integer Column: {}", catalog_col.getName());
LOG.trace("Random Integer Column: {}", catalog_col.getName().toLowerCase());
}
}
}
Expand Down Expand Up @@ -440,7 +440,7 @@ public Collection<SubTableGenerator<?>> getSubTableGenerators() {
public Collection<String> getSubGeneratorTableNames() {
List<String> names = new ArrayList<>();
for (AbstractTableGenerator gen : this.sub_generators) {
names.add(gen.catalog_tbl.getName());
names.add(gen.catalog_tbl.getName().toLowerCase());
}
return (names);
}
Expand Down Expand Up @@ -759,7 +759,7 @@ protected int populateRow(Object[] row) {
// C_ID
row[col++] = category.getCategoryID();
// C_NAME
row[col++] = category.getName();
row[col++] = category.getName().toLowerCase();
// C_PARENT_ID
row[col++] = category.getParentCategoryID();

Expand Down
29 changes: 14 additions & 15 deletions src/main/java/com/oltpbenchmark/benchmarks/seats/SEATSLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,7 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite
final boolean is_airport = catalog_tbl.getName().equalsIgnoreCase(SEATSConstants.TABLENAME_AIRPORT);

if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Generating new records for table %s [batchSize=%d]", catalog_tbl.getName(), batch_size));
LOG.debug(String.format("Generating new records for table %s [batchSize=%d]", catalog_tbl.getName().toLowerCase(), batch_size));
}
final List<Column> columns = catalog_tbl.getColumns();

Expand All @@ -515,7 +515,7 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite
Map<Integer, Map<String, Long>> mapping_columns = new HashMap<>();
for (int col_code_idx = 0, cnt = columns.size(); col_code_idx < cnt; col_code_idx++) {
Column catalog_col = columns.get(col_code_idx);
String col_name = catalog_col.getName();
String col_name = catalog_col.getName().toLowerCase();

// Code Column -> Id Column Mapping
// Check to see whether this table has columns that we need to map
Expand All @@ -529,7 +529,7 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite
code_2_id.put(col_code_idx, col_id_idx);
}


// Foreign Key Column to Code->Id Mapping
// If this columns references a foreign key that is used in the
// Code->Id mapping that we generating above,
// then we need to know when we should change the
Expand Down Expand Up @@ -592,15 +592,14 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite

long id = (Long) tuple[code_2_id.get(col_code_idx)];
if (LOG.isTraceEnabled()) {
LOG.trace(String.format("Mapping %s '%s' -> %s '%d'", from_column.getName(), code, to_column.getName(), id));
LOG.trace(String.format("Mapping %s '%s' -> %s '%d'", from_column.getName().toLowerCase(), code, to_column.getName().toLowerCase(), id));
}
this.profile.code_id_xref.get(to_column.getName()).put(code, id);
this.profile.code_id_xref.get(to_column.getName().toLowerCase()).put(code, id);
}
}


// Foreign Key Code -> Foreign Key Id
for (int col_code_idx : mapping_columns.keySet()) {
Column catalog_col = columns.get(col_code_idx);
if (tuple[col_code_idx] != null) {
String code = tuple[col_code_idx].toString();
tuple[col_code_idx] = mapping_columns.get(col_code_idx).get(code);
Expand All @@ -615,15 +614,15 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite
insert_stmt.setNull(i + 1, sqlTypes[i]);
}
} catch (SQLDataException ex) {
LOG.error("INVALID {} TUPLE: {}", catalog_tbl.getName(), Arrays.toString(tuple));
throw new RuntimeException("Failed to set value for " + catalog_tbl.getColumn(i).getName(), ex);
LOG.error("INVALID {} TUPLE: {}", catalog_tbl.getName().toLowerCase(), Arrays.toString(tuple));
throw new RuntimeException("Failed to set value for " + catalog_tbl.getColumn(i).getName().toLowerCase(), ex);
}
}
insert_stmt.addBatch();
row_idx++;

if (++row_batch >= batch_size) {
LOG.debug(String.format("Loading %s batch [total=%d]", catalog_tbl.getName(), row_idx));
LOG.debug(String.format("Loading %s batch [total=%d]", catalog_tbl.getName().toLowerCase(), row_idx));
insert_stmt.executeBatch();
insert_stmt.clearBatch();
row_batch = 0;
Expand All @@ -636,7 +635,7 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite
}

} catch (Exception ex) {
throw new RuntimeException("Failed to load table " + catalog_tbl.getName(), ex);
throw new RuntimeException("Failed to load table " + catalog_tbl.getName().toLowerCase(), ex);
}

// Record the number of tuples that we loaded for this table in the
Expand All @@ -645,7 +644,7 @@ public void loadTable(Connection conn, Table catalog_tbl, Iterable<Object[]> ite
this.profile.num_reservations = row_idx + 1;
}

LOG.info(String.format("Finished loading all %d tuples for %s", row_idx, catalog_tbl.getName()));
LOG.debug(String.format("Finished loading all %d tuples for %s", row_idx, catalog_tbl.getName().toLowerCase()));
}

// ----------------------------------------------------------------
Expand Down Expand Up @@ -678,11 +677,11 @@ public FixedDataIterable(Table catalog_tbl, String filePath) throws Exception {
// Figure out which columns are random integers and strings
for (Column catalog_col : catalog_tbl.getColumns()) {
int col_idx = catalog_col.getIndex();
if (catalog_col.getUppercaseName().contains("_SATTR")) {
if (catalog_col.getName().toUpperCase().contains("_SATTR")) {
this.rnd_string.add(col_idx);
this.rnd_string_min.put(col_idx, SEATSLoader.this.rng.nextInt(catalog_col.getSize() - 1));
this.rnd_string_max.put(col_idx, catalog_col.getSize());
} else if (catalog_col.getUppercaseName().contains("_IATTR")) {
} else if (catalog_col.getName().toUpperCase().contains("_IATTR")) {
this.rnd_integer.add(catalog_col.getIndex());
}
}
Expand Down Expand Up @@ -724,7 +723,7 @@ public Object[] next() {
* @param catalog_tbl the target table that we need an iterable for
*/
protected Iterable<Object[]> getScalingIterable(Table catalog_tbl) {
String name = catalog_tbl.getName();
String name = catalog_tbl.getName().toLowerCase();
ScalingDataIterable it = null;
double scaleFactor = this.workConf.getScaleFactor();
long num_customers = Math.round(SEATSConstants.CUSTOMERS_COUNT * scaleFactor);
Expand Down
25 changes: 12 additions & 13 deletions src/main/java/com/oltpbenchmark/benchmarks/seats/SEATSProfile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,10 @@ public SEATSProfile(SEATSBenchmark benchmark, RandomGenerator rng) {
for (Table catalog_tbl : benchmark.getCatalog().getTables()) {
for (Column catalog_col : catalog_tbl.getColumns()) {
Column catalog_fkey_col = catalog_col.getForeignKey();
if (catalog_fkey_col != null && this.code_id_xref.containsKey(catalog_fkey_col.getName())) {
this.fkey_value_xref.put(catalog_col.getName(), catalog_fkey_col.getName());
if (catalog_fkey_col != null && this.code_id_xref.containsKey(catalog_fkey_col.getName().toLowerCase())) {
this.fkey_value_xref.put(catalog_col.getName().toLowerCase(), catalog_fkey_col.getName().toLowerCase());
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Added ForeignKey mapping from %s to %s", catalog_col.getName(), catalog_fkey_col.getName()));
LOG.debug(String.format("Added ForeignKey mapping from %s to %s", catalog_col.getName().toLowerCase(), catalog_fkey_col.getName().toLowerCase()));
}
}
}
Expand Down Expand Up @@ -297,20 +297,19 @@ protected final void loadProfile(SEATSWorker worker) throws SQLException {
try (Connection conn = benchmark.getConnection()) {
results = proc.run(conn);
}
// CONFIG_PROFILE
this.loadConfigProfile(results.getConfigProfile());
// CONFIG_PROFILE
this.loadConfigProfile(results.getConfigProfile());

// CONFIG_HISTOGRAMS
this.loadConfigHistograms(results.getConfigHistogram());
// CONFIG_HISTOGRAMS
this.loadConfigHistograms(results.getConfigHistogram());


this.loadCodeXref(results.getCountryCodes(), SEATSConstants.COUNTRY_CODE, SEATSConstants.COUNTRY_ID);
this.loadCodeXref(results.getAirportCodes(), SEATSConstants.AIRPORT_CODE, SEATSConstants.AIRPORT_ID);
this.loadCodeXref(results.getAirlineCodes(), SEATSConstants.AIRLINE_IATA_CODE, SEATSConstants.AIRLINE_ID);

// CACHED FLIGHT IDS
this.loadCachedFlights(results.getFlights());
this.loadCodeXref(results.getCountryCodes(), SEATSConstants.COUNTRY_CODE, SEATSConstants.COUNTRY_ID);
this.loadCodeXref(results.getAirportCodes(), SEATSConstants.AIRPORT_CODE, SEATSConstants.AIRPORT_ID);
this.loadCodeXref(results.getAirlineCodes(), SEATSConstants.AIRLINE_IATA_CODE, SEATSConstants.AIRLINE_ID);

// CACHED FLIGHT IDS
this.loadCachedFlights(results.getFlights());


if (LOG.isDebugEnabled()) {
Expand Down
13 changes: 8 additions & 5 deletions src/main/java/com/oltpbenchmark/benchmarks/tatp/TATPLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ void genSpeAndCal(Connection conn) throws SQLException {

try (PreparedStatement cal_pstmt = conn.prepareStatement(cal_sql);
PreparedStatement spe_pstmt = conn.prepareStatement(spe_sql)) {
boolean cal_added = false;
while (s_id++ < subscriberSize) {
int[] sf_types = TATPUtil.subArr(spe_arr, 1, 4);
for (int sf_type : sf_types) {
Expand All @@ -263,6 +264,7 @@ void genSpeAndCal(Connection conn) throws SQLException {
cal_pstmt.setByte(++cal_col, (byte) (start_time + TATPUtil.number(1, 8)));
cal_pstmt.setString(++cal_col, TATPUtil.nstring(15, 15));
cal_pstmt.addBatch();
cal_added = true;
cal_total++;
}
}
Expand All @@ -273,12 +275,13 @@ void genSpeAndCal(Connection conn) throws SQLException {
}
int[] results = spe_pstmt.executeBatch();


if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s: %d (%s %d / %d)", TATPConstants.TABLENAME_CALL_FORWARDING, cal_total, TATPConstants.TABLENAME_SUBSCRIBER, s_id, subscriberSize));
if (cal_added) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("%s: %d (%s %d / %d)", TATPConstants.TABLENAME_CALL_FORWARDING, cal_total, TATPConstants.TABLENAME_SUBSCRIBER, s_id, subscriberSize));
}
results = cal_pstmt.executeBatch();
cal_added = false;
}
results = cal_pstmt.executeBatch();


spe_batch = 0;
}
Expand Down
13 changes: 3 additions & 10 deletions src/main/java/com/oltpbenchmark/catalog/AbstractCatalogObject.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,13 @@ public abstract class AbstractCatalogObject implements Serializable {
static final long serialVersionUID = 0;

protected final String name;
protected final String uppercaseName;
protected final String separator;

public AbstractCatalogObject(String name, String uppercaseName, String separator) {
public AbstractCatalogObject(String name, String separator) {
this.name = name;
this.uppercaseName = uppercaseName;
this.separator = separator;
}

public String getUppercaseName() {
return uppercaseName;
}

public String getName() {
return name;
}
Expand Down Expand Up @@ -69,13 +63,12 @@ public boolean equals(Object o) {
return false;
}
AbstractCatalogObject that = (AbstractCatalogObject) o;
return Objects.equals(uppercaseName, that.uppercaseName) &&
Objects.equals(name, that.name) &&
return Objects.equals(name, that.name) &&
Objects.equals(separator, that.separator);
}

@Override
public int hashCode() {
return Objects.hash(uppercaseName, name, separator);
return Objects.hash(name, separator);
}
}
4 changes: 2 additions & 2 deletions src/main/java/com/oltpbenchmark/catalog/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ public class Column extends AbstractCatalogObject {

private Column foreignKey = null;

public Column(String name, String uppercaseName, String separator, Table table, int type, Integer size, boolean nullable) {
super(name, uppercaseName, separator);
public Column(String name, String separator, Table table, int type, Integer size, boolean nullable) {
super(name, separator);
this.table = table;
this.type = type;
this.size = size;
Expand Down
Loading