Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert multiple files at same time #41

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Convert multiple files at same time #41

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 75 additions & 66 deletions src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
import { promises as fs ,existsSync} from "fs";
import {createIfNot} from "./utils/fileUtils.js"
import * as path from "node:path"
import { promises as fs, existsSync } from "fs";
import { createIfNot } from "./utils/fileUtils.js";
import * as path from "node:path";

async function writeSQL(statement: string, saveFileAs = "", isAppend: boolean = false) {
try {
const destinationFile = process.argv[2] || saveFileAs;
const destinationFile = saveFileAs;
if (!destinationFile) {
throw new Error("Missing saveFileAs parameter");
}
createIfNot(path.resolve(`./sql/${destinationFile}.sql`))
if(isAppend){
await fs.appendFile(`sql/${process.argv[2]}.sql`, statement);
}else{
await fs.writeFile(`sql/${process.argv[2]}.sql`, statement);
createIfNot(path.resolve(`../sql/${destinationFile}.sql`));
if (isAppend) {
await fs.appendFile(`sql/${destinationFile}.sql`, statement);
} else {
await fs.writeFile(`sql/${destinationFile}.sql`, statement);
}
} catch (err) {
console.log(err);
Expand All @@ -21,75 +22,83 @@ async function writeSQL(statement: string, saveFileAs = "", isAppend: boolean =
async function readCSV(csvFileName = "", batchSize: number = 0) {
try {
const fileAndTableName = process.argv[2] || csvFileName;

batchSize = parseInt(process.argv[3]) || batchSize || 500;
let isAppend: boolean = false;
let isAppend: boolean = false;

if (!fileAndTableName) {
throw new Error("Missing csvFileName parameter");
}
if(!existsSync(path.resolve(`./csv/${fileAndTableName}.csv`))){
console.log("file not found")
return
}
const data = await fs.readFile(`csv/${fileAndTableName}.csv`, {
encoding: "utf8",
});
const linesArray = data.split(/\r|\n/).filter((line) => line);
const columnNames = linesArray?.shift()?.split(",") || [];
let beginSQLInsert = `INSERT INTO ${fileAndTableName} (`;
columnNames.forEach((name) => (beginSQLInsert += `${name}, `));
beginSQLInsert = beginSQLInsert.slice(0, -2) + ")\nVALUES\n";
let values = "";
linesArray.forEach((line, index) => {
// Parses each line of CSV into field values array
const arr = line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/);
if (arr.length > columnNames.length) {
console.log(arr);
throw new Error("Too Many Values in row");
} else if (arr.length < columnNames.length) {
console.log(arr);
throw new Error("Too Few Values in row");
}

// Check batch size (rows per batch)
if(index > 0 && index % batchSize == 0){
values = values.slice(0, -2) + ";\n\n";

const sqlStatement = beginSQLInsert + values;

// Write File
writeSQL(sqlStatement, fileAndTableName, isAppend);
values = "";
isAppend = true;
const csvFiles = fileAndTableName.split(",");

for (const csvFile of csvFiles) {
if (!existsSync(path.resolve(`./csv/${csvFile}.csv`))) {
console.log(`File not found: ${csvFile}`);
continue;
}

let valueLine = "\t(";
arr.forEach((value) => {
// Matches NULL values, Numbers,
// Strings accepted as numbers, and Booleans (0 or 1)
if (value === "NULL" || !isNaN(+value)) {
valueLine += `${value}, `;
} else {
// If a string is wrapped in quotes, it doesn't need more
if (value.at(0) === '"') valueLine += `${value}, `;
else {
// This wraps strings in quotes
// also wraps timestamps
valueLine += `"${value}", `;
}

const data = await fs.readFile(`csv/${csvFile}.csv`, {
encoding: "utf8",
});
const linesArray = data.split(/\r|\n/).filter((line) => line);
const columnNames = linesArray?.shift()?.split(",") || [];
let beginSQLInsert = `INSERT INTO ${csvFile} (`;
columnNames.forEach((name) => (beginSQLInsert += `${name}, `));
beginSQLInsert = beginSQLInsert.slice(0, -2) + ")\nVALUES\n";
let values = "";
linesArray.forEach((line, index) => {
// Parses each line of CSV into field values array
const arr = line.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/);
if (arr.length > columnNames.length) {
console.log(arr);
throw new Error("Too Many Values in row");
} else if (arr.length < columnNames.length) {
console.log(arr);
throw new Error("Too Few Values in row");
}

// Check batch size (rows per batch)
if (index > 0 && index % batchSize == 0) {
values = values.slice(0, -2) + ";\n\n";

const sqlStatement = beginSQLInsert + values;

// Write File
writeSQL(sqlStatement, csvFile, isAppend);
values = "";
isAppend = true;
}

let valueLine = "\t(";
arr.forEach((value) => {
// Matches NULL values, Numbers,
// Strings accepted as numbers, and Booleans (0 or 1)
if (value === "NULL" || !isNaN(+value)) {
valueLine += `${value}, `;
} else {
// If a string is wrapped in quotes, it doesn't need more
if (value.at(0) === '"') valueLine += `${value}, `;
else {
// This wraps strings in quotes
// also wraps timestamps
valueLine += `"${value}", `;
}
}
});
valueLine = valueLine.slice(0, -2) + "),\n";
values += valueLine;
});
valueLine = valueLine.slice(0, -2) + "),\n";
values += valueLine;
});
values = values.slice(0, -2) + ";";
const sqlStatement = beginSQLInsert + values;
// Write File
writeSQL(sqlStatement, fileAndTableName, isAppend);
values = values.slice(0, -2) + ";";
const sqlStatement = beginSQLInsert + values;
// Write File
writeSQL(sqlStatement, csvFile, isAppend);
}
} catch (err) {
console.log(err);
}
}

// Usage example: node yourScript.js file1,file2,file3
readCSV();
console.log("Finished!");