data & perftest

crawler/importListing.ts

@@ -1,19 +1,31 @@
+import yargs from 'yargs'
+import { hideBin } from 'yargs/helpers'
 import fs from 'fs-extra';
 import { selectFiles } from './filechooser.js';
 import path from 'path';
 
+const argv = yargs(hideBin(process.argv)).argv
+
+
 (async () => {
     const result = await selectFiles({startingPath:'./data',directoryFilter: (directoryName)=>{
         return false;
       },fileFilter: (fileName) => {
         return /\.json$/gi.test(fileName);
       },pageSize:10,multi:false});
-      console.log(result['selectedFiles'][0]);
-      const file = result['selectedFiles'][0];
-    //   const extension = path.extname(file);
-    //   const basefileName = path.basename(file,extension);      
+    if (argv.delete){ // delete==business or commercialProperty
+      const ids = await fetch(`http://localhost:3000/bizmatch/listings/undefined/repo/${argv.delete}`,{
+        method: 'GET',
+        headers: { 'Content-Type': 'application/json' },
+      })
+      await fetch(`http://localhost:3000/bizmatch/listings/undefined/repo/${argv.delete}`,{
+        method: 'DELETE',
+        headers: { 'Content-Type': 'application/json' },
+      })        
+    }
+    console.log(result['selectedFiles'][0]);
+    const file = result['selectedFiles'][0];
     const listings = await fs.readJson(file);
-    //listings.forEach(element => {
     for (const listing of listings) {
         const type = listing.listingsCategory
         const response = await fetch(`http://localhost:3000/bizmatch/listings/${type}`, {

crawler/package.json

@@ -20,6 +20,8 @@
     "ioredis": "^5.3.2",
     "node-fetch": "^3.3.2",
     "puppeteer": "^22.1.0",
+    "redis": "^4.6.13",
+    "winston": "^3.13.0",
     "yargs": "^17.7.2"
   }
 }

crawler/perftest.ts

@@ -0,0 +1,24 @@
+import  {createLogger,transports, format}  from 'winston';
+import { createClient } from 'redis';
+const logger = createLogger({
+    transports: [
+      new transports.Console(),
+    ],
+    format: format.combine(
+        format.colorize(),
+        format.timestamp(),
+        format.printf(({ timestamp, level, message, service }) => {
+          return `[${timestamp}] ${service} ${level}: ${message}`;
+        })
+      ),
+});
+const redis = await createClient()
+.on('error', err => console.log('Redis Client Error', err))
+.connect();
+
+(async () => {
+    logger.info(`start findBusinessListings:`);
+    const result = await redis.ft.search('business:index','*',{LIMIT:{from:0,size:50}});
+    logger.info(`end findBusinessListings:`);
+    redis.disconnect();
+})();