From e68612ed1bda648adcfe301f0a226eaffee78732 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 20 Aug 2022 12:29:23 +0530 Subject: [PATCH 01/34] Switch to ES Modules --- bot.js | 12 ++++++------ config.js | 18 +++++++++--------- db.js | 6 +++--- index.js | 10 +++++----- package.json | 1 + unshort.js | 4 ++-- utils.js | 8 ++++---- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/bot.js b/bot.js index f9358f4..8266ed6 100644 --- a/bot.js +++ b/bot.js @@ -1,9 +1,9 @@ // Imports -const { Bot } = require("grammy"); -const { BOT_TOKEN, ADMINS, LIMIT } = require("./config"); -const { isUrl, getRandomId, getProductDetails } = require("./utils"); -const { manageProducts, manageUsers } = require("./db"); -const unshort = require("./unshort"); +import { Bot } from "grammy" +import { BOT_TOKEN, ADMINS, LIMIT } from "./config.js" +import { isUrl, getRandomId, getProductDetails } from "./utils.js" +import { manageProducts, manageUsers } from "./db.js" +import unshort from "./unshort.js" const bot = new Bot(BOT_TOKEN); // Initialize bot @@ -257,4 +257,4 @@ bot.catch((err) => { setInterval(track, 3600000); //Track every hr. -module.exports = bot; +export default bot; diff --git a/config.js b/config.js index cf03711..bead029 100644 --- a/config.js +++ b/config.js @@ -1,4 +1,4 @@ -require('dotenv').config() +import 'dotenv/config' // Check if bot token is there or not if(!process.env.BOT_TOKEN) { @@ -11,11 +11,11 @@ if(!process.env.DB_URL) { process.exit(1); } -module.exports = { - ADMINS: process.env.ADMINS || '', - BOT_TOKEN: process.env.BOT_TOKEN || '', - DB_URL: process.env.DB_URL || '', - WORKER_URL: process.env.WORKER_URL || '', - API_KEY: process.env.API_KEY || '', // Generate any API Key and pass it when accessing the API. - LIMIT: Number(process.env.LIMIT), // Maximum number of products can be added by a user at a time. -} \ No newline at end of file +const ADMINS = process.env.ADMINS || '' +const BOT_TOKEN = process.env.BOT_TOKEN || '' +const DB_URL = process.env.DB_URL || '' +const WORKER_URL = process.env.WORKER_URL || '' +const API_KEY = process.env.API_KEY || '' // Generate any API Key and pass it when accessing the API. +const LIMIT = Number(process.env.LIMIT) // Maximum number of products can be added by a user at a time. + +export { ADMINS, BOT_TOKEN, DB_URL, WORKER_URL, API_KEY, LIMIT } \ No newline at end of file diff --git a/db.js b/db.js index 76919fb..3021c3e 100644 --- a/db.js +++ b/db.js @@ -1,5 +1,5 @@ -const {DB_URL} = require('./config'); -const MongoClient = require('mongodb').MongoClient; +import {DB_URL} from './config.js'; +import {MongoClient} from 'mongodb'; var mongo; // Check if mongodb is connected or not @@ -65,4 +65,4 @@ const manageProducts = async(data, action) => { } } -module.exports = {manageProducts, manageUsers}; \ No newline at end of file +export {manageProducts, manageUsers}; \ No newline at end of file diff --git a/index.js b/index.js index e69e41e..34f7ad6 100644 --- a/index.js +++ b/index.js @@ -1,8 +1,8 @@ -const { getRandomId, getProductDetails } = require("./utils"); -const { manageProducts, manageUsers } = require("./db"); -const {API_KEY} = require('./config'); -const express = require('express'); -const bot = require('./bot'); +import { getRandomId, getProductDetails } from "./utils.js" +import { manageProducts, manageUsers } from "./db.js" +import {API_KEY} from './config.js' +import express from 'express' +import bot from './bot.js' //Globals const port = process.env.PORT || 3000; diff --git a/package.json b/package.json index f1adac1..e4fda66 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "type": "git", "url": "git+https://github.com/AffanTheBest/price-tracker.git" }, + "type": "module", "keywords": [ "price-tracker", "amazon", diff --git a/unshort.js b/unshort.js index bb892e1..ca65765 100644 --- a/unshort.js +++ b/unshort.js @@ -1,4 +1,4 @@ -const axios = require('axios'); +import axios from 'axios'; const unshort = async (url) => { const extractUrl = req => req?.request?.res?.responseUrl || req?.request?._redirectable?._currentUrl || @@ -16,4 +16,4 @@ const unshort = async (url) => { return longUrl; } -module.exports = unshort; \ No newline at end of file +export default unshort; \ No newline at end of file diff --git a/utils.js b/utils.js index d04059a..d234709 100644 --- a/utils.js +++ b/utils.js @@ -1,6 +1,6 @@ -const cheerio = require('cheerio'); -const axios = require('axios'); -const {WORKER_URL} = require('./config'); +import cheerio from 'cheerio' +import axios from 'axios' +import {WORKER_URL} from './config.js' const urlRegex = /^(?:(?:(?:https?|ftp):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:[/?#]\S*)?$/i; @@ -54,4 +54,4 @@ const getProductDetails = async(url, merchant) => { } } -module.exports = { isUrl, getRandomId, getProductDetails }; \ No newline at end of file +export { isUrl, getRandomId, getProductDetails }; \ No newline at end of file From 5e2c0adc2387e69d506175f6d86ff6cd47dafa7b Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 20 Aug 2022 12:50:44 +0530 Subject: [PATCH 02/34] Add deployment methods --- README.md | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0feff89..fe3e7c5 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,12 @@ If you like this project, please leave a 🌟. --- +- [Features](#features) +- [Usage](#how-to-use) +- [Contributing](#contributing) + +--- + ## Features ``` @@ -31,14 +37,20 @@ If you like this project, please leave a 🌟. --- -## NOTE +## Deploy -``` -- Do not clone this repo (You can fork it instead). -- Use this bot at your own risk. -- This bot can be a little bit slow cuz using cloudfare workers for scrapping product data (To avoid ip blocking from Amazon & Flipkart) -``` +- [![Deploy with Heroku](https://www.herokucdn.com/deploy/button.svg "Deploy with Heroku")](https://heroku.com/deploy?template=https://github.com/siddiquiaffan/price-tracker "Deploy with Heroku") + +- [![Deploy+on+Railway](https://railway.app/button.svg)](https://railway.app/new/template?template=https://github.com/siddiquiaffan/price-tracker&envs=ADMINS,BOT_TOKEN,DB_URL,WORKER_URL,API_KEY,LIMIT&ADMINSDesc=Telegarm+ids+of+admins+separated+by+space&BOT_TOKENDesc=Get+Your+Bot+Token+From+@BotFather.&DB_URLDesc=Create+A+Database+In+Mongodb+And+Get+URL.&WORKER_URLDesc=Paste+worker.js+code+in+Cloudfare+Worker+and+get+url.&API_KEYDesc=Any+secret+key+to+access+API&LIMITDesc=Limit+of+products+to+track+per+user.) +- ### Local - + + ``` + - Clone Repository + - Install Dependencies (npm install) + - Create .env file and fill it with your details. + - Start App (npm start) + ``` --- ## Contributing @@ -51,6 +63,16 @@ If you like this project, please leave a 🌟. --- +## NOTE + +``` +- Do not clone this repo (You can fork it instead). +- Use this bot at your own risk. +- This bot can be a little bit slow cuz using cloudfare workers for scrapping product data (To avoid ip blocking from Amazon & Flipkart) +``` + +--- + ## License [LICENSE](https://github.com/siddiquiaffan/price-tracker/blob/main/LICENSE) From c67f0399c9facf168c60b9ccdef5a92f9d5f7682 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 20 Aug 2022 12:54:13 +0530 Subject: [PATCH 03/34] Update TOC --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fe3e7c5..60abab5 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,9 @@ If you like this project, please leave a 🌟. - [Features](#features) - [Usage](#how-to-use) +- [Deployment](#deploy) - [Contributing](#contributing) - +- --- ## Features From 6b028763452445aafdcb7c5114e5b3a6ef59ea00 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Mon, 22 Aug 2022 17:29:11 +0530 Subject: [PATCH 04/34] Improve performance Merge all users tracking same produt to reduce the load --- bot.js | 40 ++++++++++++++++++++++------------------ db.js | 26 ++++++++++++++++++++++++-- utils.js | 32 ++++++++++++++++++++++++++------ 3 files changed, 72 insertions(+), 26 deletions(-) diff --git a/bot.js b/bot.js index 8266ed6..9e52e09 100644 --- a/bot.js +++ b/bot.js @@ -184,8 +184,10 @@ bot.command("users", async (ctx) => { bot.command("stats", async (ctx) => { const users = await manageUsers({}, "read"); const products = await manageProducts({}, "read"); + let userCount = 0; + products.result.map((p) => (p += item.users.length)); ctx.reply( - `Total Users: ${users.result.length}\nTotal Products: ${products.result.length}` + `Total Users: ${users.result.length}\nTotal Products: ${userCount}` ); }); @@ -222,23 +224,25 @@ const track = async () => { const details = await getProductDetails(product.link, product.merchant); if (details.ok && !isNaN(details.price) && details.price !== product.price) { try { - await manageProducts({ tracking_id: product.tracking_id, userId: product.userId, merchant: product.merchant, title: details.title, link: product.link, initPrice: product.price, price: details.price, }, "update"); - bot.api.sendMessage( - product.userId, - ` Price has been ${details.price > product.price ? "increased" : "decreased" - } by ${Math.abs(product.price - details.price)}. \n\n${details.title - }\n\nCurrent Price: ${details.price}\nLink: ${product.merchant}\n\nTo stop tracking send /stop_${product.tracking_id - }`, - { - parse_mode: "HTML", - reply_markup: { - inline_keyboard: details?.link ? [ - [{ text: "Buy Now", url: details.link }], - [{ text: "Stop Tracking - " + product.tracking_id, callback_data: `stopTracking`, }]] - : [] - } - }); + await manageProducts({ tracking_id: product.tracking_id, userId: product.userId, merchant: product.merchant, title: details.title, link: product.link, initPrice: product.price, price: details.price, users: product.users}, "update"); + await Promise.all(product.users.map(async user => { + bot.api.sendMessage( + user.userId, + ` Price has been ${details.price > product.price ? "increased" : "decreased" + } by ${Math.abs(product.price - details.price)}. \n\n${details.title + }\n\nCurrent Price: ${details.price}\nLink: ${product.merchant}\n\nTo stop tracking send /stop_${user.tracking_id + }`, + { + parse_mode: "HTML", + reply_markup: { + inline_keyboard: details?.link ? [ + [{ text: "Buy Now", url: details.link }], + [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] + : [] + } + }); + })) } catch (e) { bot.start() } } }) diff --git a/db.js b/db.js index 3021c3e..8dfffb1 100644 --- a/db.js +++ b/db.js @@ -48,10 +48,26 @@ const manageProducts = async(data, action) => { const collection = db.collection('tasks'); switch(action) { case 'delete': - await collection.deleteOne({tracking_id: data.tracking_id, userId: data.userId}); + await collection.updateOne( + { users: {userId: data.userId, tracking_id: data.tracking_id } }, + { $pull: { users : {userId: data.userId, tracking_id: data.tracking_id } }, } + ); return {ok: true} case 'update': - const res = await collection.updateOne({tracking_id: data.tracking_id}, {$set: data}, {upsert: true}); + await collection.updateOne( + { link: data.link }, + { + $set: { + link: data.link, + merchant: data.merchant, + initPrice: data.initPrice, + price: data.price, + title: data.title, + }, + $addToSet: {users : Array.isArray(data.users) ? {$each: data.users} : data.users}, + }, + { upsert: true } + ); return {ok: true, tracking_id: data.tracking_id} case 'read': const result = await collection.find(data).toArray(); @@ -65,4 +81,10 @@ const manageProducts = async(data, action) => { } } +// manageProducts({}, 'read').then(data => { +// console.log(data); +// fs.writeFileSync('products.json', JSON.stringify(data.result)); +// }).catch(e => { +// }) + export {manageProducts, manageUsers}; \ No newline at end of file diff --git a/utils.js b/utils.js index d234709..46a5fc8 100644 --- a/utils.js +++ b/utils.js @@ -28,9 +28,32 @@ const selectors = { } } +const productCommonUrl = (link) => { + const url = new URL(link?.replace("www.", "")); + const merchant = url.hostname.split(".")[0]; + let id, commonUrl; + switch (merchant) { + case "amazon": + id = link.match( + /https?:\/\/(www\.)?(.*)amazon\.([a-z\.]{2,6})(\/d\/(.*)|\/(.*)\/?(?:dp|o|gp|-)\/)(aw\/d\/|product\/)?(B[0-9]{1}[0-9A-Z]{8}|[0-9]{9}(?:X|[0-9]))/i + ).splice(-1)[0]; + commonUrl = "https://www.amazon.in/dp/" + id + "?tag=asloot-21"; + break; + case "flipkart": + id = url.searchParams.get("pid"); + commonUrl = id ? "https://www.flipkart.com/product/p/itme?pid=" + id : link.includes('/p/itm') ?link.split('?')[0] : link; + break; + default: + null; + } + + return commonUrl; +}; + const getProductDetails = async(url, merchant) => { try{ - const res = await axios.get(`${WORKER_URL}/?url=${encodeURIComponent(url)}`, { + const commonUrl = productCommonUrl(url); + const res = await axios.get(`${WORKER_URL}/?url=${encodeURIComponent(commonUrl)}`, { headers: { "User-Agent": "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36", @@ -38,20 +61,17 @@ const getProductDetails = async(url, merchant) => { }); const $ = cheerio.load(res.data); const selector = selectors[merchant]; - let link = new URL(url); - if(merchant == 'amazon') link.searchParams.set('tag', 'asloot-21'); - link = link.toString(); const price = parseInt($(selector.price1).text().trim().replace(/^\D+|[^0-9.]/g, '')) || parseInt($(selector.price2).text().trim().replace(/^\D+|[^0-9.]/g, '')); const title = $(selector.title).text().trim(); const image = $(selector.image1).attr('src'); if(!title || !price) { return {ok: false} } - return {ok: true, title, price, image, link} + return {ok: true, title, price, image, link: commonUrl} }catch(e){ console.log(e); return {ok: false} } } -export { isUrl, getRandomId, getProductDetails }; \ No newline at end of file +export { isUrl, getRandomId, getProductDetails, productCommonUrl }; \ No newline at end of file From ae1e2f664e69a923349d3ed962a80d3899e802f9 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 12:47:09 +0530 Subject: [PATCH 05/34] Update mongo client --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e4fda66..2e3bdbc 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,6 @@ "dotenv": "^10.0.0", "express": "^4.17.1", "grammy": "^1.5.4", - "mongodb": "^4.1.3" + "mongodb": "^4.9.0" } } From 490dbc741eec011f0a03daa4736577d75b4affa1 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 12:47:55 +0530 Subject: [PATCH 06/34] Don't let users add duplidate products --- db.js | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/db.js b/db.js index 8dfffb1..bc3a3f5 100644 --- a/db.js +++ b/db.js @@ -41,10 +41,18 @@ const manageUsers = async(data, action) => { } } +const addToSet = (data) => { + if(Array.isArray(data.users)) + return({$addToSet: { users: {$each: data.users}}}) + else + return({$exists: false}, {$addToSet: { users: data.users}}) +} + const manageProducts = async(data, action) => { await connectDb(); try{ - const db = mongo.db('AS_TRACKER'); + console.log(data.userId) + const db = mongo.db('TESTS'); const collection = db.collection('tasks'); switch(action) { case 'delete': @@ -55,18 +63,20 @@ const manageProducts = async(data, action) => { return {ok: true} case 'update': await collection.updateOne( - { link: data.link }, - { - $set: { - link: data.link, - merchant: data.merchant, - initPrice: data.initPrice, - price: data.price, - title: data.title, - }, - $addToSet: {users : Array.isArray(data.users) ? {$each: data.users} : data.users}, - }, - { upsert: true } + { link: data.link }, + [ + { $set: { + link: data.link, merchant: data.merchant, initPrice: data.initPrice, price: data.price, title: data.title, + users: {$concatArrays: [ + {$ifNull: ["$users", []]}, + {$filter: { + input: data.users || [{userId: data.userId, tracking_id: data.tracking_id }], + cond: {$not: {$in: ['$$this.userId', {$ifNull: ["$users.userId", []]}]}} + }} + ]} + }}, + ], + { upsert: true } ); return {ok: true, tracking_id: data.tracking_id} case 'read': From 9bc88b1315a3f6f6e76b36c607afae1cd332252c Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 12:55:15 +0530 Subject: [PATCH 07/34] Remove log --- db.js | 1 - 1 file changed, 1 deletion(-) diff --git a/db.js b/db.js index bc3a3f5..9b183a6 100644 --- a/db.js +++ b/db.js @@ -51,7 +51,6 @@ const addToSet = (data) => { const manageProducts = async(data, action) => { await connectDb(); try{ - console.log(data.userId) const db = mongo.db('TESTS'); const collection = db.collection('tasks'); switch(action) { From c7d52f92f17d693b55e8db051bcac8d67453bb6e Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 13:02:31 +0530 Subject: [PATCH 08/34] Remove unsed code --- db.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/db.js b/db.js index 9b183a6..33d9596 100644 --- a/db.js +++ b/db.js @@ -41,13 +41,6 @@ const manageUsers = async(data, action) => { } } -const addToSet = (data) => { - if(Array.isArray(data.users)) - return({$addToSet: { users: {$each: data.users}}}) - else - return({$exists: false}, {$addToSet: { users: data.users}}) -} - const manageProducts = async(data, action) => { await connectDb(); try{ From e72db7dcdc37d3bfff9e4775e0ae0ee3f7ace122 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 13:03:09 +0530 Subject: [PATCH 09/34] query to main db --- db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db.js b/db.js index 33d9596..1abec53 100644 --- a/db.js +++ b/db.js @@ -44,7 +44,7 @@ const manageUsers = async(data, action) => { const manageProducts = async(data, action) => { await connectDb(); try{ - const db = mongo.db('TESTS'); + const db = mongo.db('AS_TRACKER'); const collection = db.collection('tasks'); switch(action) { case 'delete': From 61fa34be529e6b284853073f0e7db50b5ccf634f Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 13:04:09 +0530 Subject: [PATCH 10/34] Update stats acc to db changes --- index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 34f7ad6..0c129dc 100644 --- a/index.js +++ b/index.js @@ -63,7 +63,9 @@ app.get('/stats', async (req, res) => { try{ const users = await manageUsers({}, 'read'); const products = await manageProducts({}, 'read'); - res.status(200).send(JSON.stringify({users: users?.result?.length, products: products?.result?.length})) + let productCount = 0; + products.result.map((p) => (p += item.users.length)); + res.status(200).send(JSON.stringify({users: users?.result?.length, products: productCount })) }catch(e){ } From e87a855a06c026835ce7c710f6151ac5a5957e0c Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 13:04:36 +0530 Subject: [PATCH 11/34] Update tracking list acc to db change --- bot.js | 8 +++--- zzzz.js | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 zzzz.js diff --git a/bot.js b/bot.js index 9e52e09..26ed956 100644 --- a/bot.js +++ b/bot.js @@ -104,11 +104,11 @@ bot.command("track", async (ctx) => { bot.command("list", async (ctx) => { try { - const products = await manageProducts({ userId: ctx.from.id }, "read"); + const products = await manageProducts({ 'users.userId': ctx.from.id }, "read"); const list = products.result .map( (product) => - `${product.title}\nLast Price: ${product.price}\nLink: ${product.merchant}\nTo stop send /stop_${product.tracking_id}` + `${product.title}\nLast Price: ${product.price}\nLink: ${product.merchant}\nTo stop send /stop_${product.users.filter(u => u.userId == ctx.from.id)[0].tracking_id}` ) .join("\n\n"); ctx.reply(`Here is your tracking list:\n\n${list}`, { @@ -184,10 +184,10 @@ bot.command("users", async (ctx) => { bot.command("stats", async (ctx) => { const users = await manageUsers({}, "read"); const products = await manageProducts({}, "read"); - let userCount = 0; + let productCount = 0; products.result.map((p) => (p += item.users.length)); ctx.reply( - `Total Users: ${users.result.length}\nTotal Products: ${userCount}` + `Total Users: ${users.result.length}\nTotal Products: ${productCount}` ); }); diff --git a/zzzz.js b/zzzz.js new file mode 100644 index 0000000..42b2a58 --- /dev/null +++ b/zzzz.js @@ -0,0 +1,75 @@ +import {manageProducts} from './db.js' +import { productCommonUrl } from './utils.js' +import fs from 'fs' +import { DB_URL } from "./config.js"; +import { MongoClient } from "mongodb"; +var mongo; + +// Check if mongodb is connected or not +const isDbConnected = () => + !!mongo && !!mongo.topology && mongo.topology.isConnected(); + +// Connect to mongodb +const connectDb = async () => { + if (!isDbConnected()) { + try { + mongo = await MongoClient.connect(DB_URL, { + useNewUrlParser: true, + useUnifiedTopology: true, + }); + console.log("Connected to Database!"); + } catch (e) { + console.log("Failed to connected to Database!"); + } + } +}; + + +const func = async () => { + await connectDb(); + const collection = mongo.db("AS_TRACKER").collection("tasks"); + const result = await collection.find({}).toArray(); + console.log(result[0].users.length); + let count = 0; + result.map((item) => { + count += item.users.length + }); + + console.log(count); +} +// let data = await manageProducts({}, 'read') +// data.result.map((product,i) => { +// const url = productCommonUrl(product.link); +// data.result[i].link = url +// }) +// fs.writeFileSync('updated.json', JSON.stringify(data.result)); +// const data = fs.readFileSync('updated.json').toString(); +// let products = JSON.parse(data); +// let arr = []; +// const duplicate = products.filter(product => { +// const same = products.filter(p => { +// return p.link === product.link +// }) +// const users = same.map((p) => ({ userId: p.userId, tracking_id: p.tracking_id})); +// arr.push({ +// link: product.link, +// merchant: product.merchant, +// initPrice: product.initPrice, +// price: product.price, +// title: product.title, +// users + +// }) +// // same.map((p, index) => { +// // delete products[index] +// // }) +// }) +// fs.writeFileSync('duplicate.json', JSON.stringify(arr)); + +// (async () => await func())() +const deleteData = async () => { + await connectDb(); + const db = mongo.db('TESTS').collection('tasks') + await db.deleteOne({users: null}) +} +(async () => await deleteData())() \ No newline at end of file From 04b7e9eac9d453969d5aa9ae1b7cda63d3fcd74b Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 15:12:23 +0530 Subject: [PATCH 12/34] Fix stats command --- bot.js | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/bot.js b/bot.js index 26ed956..f8bd0fb 100644 --- a/bot.js +++ b/bot.js @@ -182,13 +182,18 @@ bot.command("users", async (ctx) => { }); bot.command("stats", async (ctx) => { - const users = await manageUsers({}, "read"); - const products = await manageProducts({}, "read"); - let productCount = 0; - products.result.map((p) => (p += item.users.length)); - ctx.reply( - `Total Users: ${users.result.length}\nTotal Products: ${productCount}` - ); + try{ + const[users, products] = await Promise.all([manageUsers, manageProducts].map( + async (func) => await func({}, "read") + )); + let prodCount = 0; + products.result.map(prod => prodCount += prod.users.length); + ctx.reply( + `Total Users: ${users.result.length}\nTotal Products: ${prodCount}` + ); + }catch(e){ + console.log(e) + } }); bot.on('::url', async ctx => { From 201694c30b95bcf643c676f3a681400f2f7e7aea Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 24 Aug 2022 15:12:39 +0530 Subject: [PATCH 13/34] Remove unused code --- db.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/db.js b/db.js index 1abec53..50f4245 100644 --- a/db.js +++ b/db.js @@ -83,10 +83,4 @@ const manageProducts = async(data, action) => { } } -// manageProducts({}, 'read').then(data => { -// console.log(data); -// fs.writeFileSync('products.json', JSON.stringify(data.result)); -// }).catch(e => { -// }) - export {manageProducts, manageUsers}; \ No newline at end of file From bd75fdc0488408d3b9a3a51e178017e67d8f56ec Mon Sep 17 00:00:00 2001 From: Siddiqui Affan Date: Sun, 18 Sep 2022 17:17:56 +0530 Subject: [PATCH 14/34] delete --- zzzz.js | 75 --------------------------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 zzzz.js diff --git a/zzzz.js b/zzzz.js deleted file mode 100644 index 42b2a58..0000000 --- a/zzzz.js +++ /dev/null @@ -1,75 +0,0 @@ -import {manageProducts} from './db.js' -import { productCommonUrl } from './utils.js' -import fs from 'fs' -import { DB_URL } from "./config.js"; -import { MongoClient } from "mongodb"; -var mongo; - -// Check if mongodb is connected or not -const isDbConnected = () => - !!mongo && !!mongo.topology && mongo.topology.isConnected(); - -// Connect to mongodb -const connectDb = async () => { - if (!isDbConnected()) { - try { - mongo = await MongoClient.connect(DB_URL, { - useNewUrlParser: true, - useUnifiedTopology: true, - }); - console.log("Connected to Database!"); - } catch (e) { - console.log("Failed to connected to Database!"); - } - } -}; - - -const func = async () => { - await connectDb(); - const collection = mongo.db("AS_TRACKER").collection("tasks"); - const result = await collection.find({}).toArray(); - console.log(result[0].users.length); - let count = 0; - result.map((item) => { - count += item.users.length - }); - - console.log(count); -} -// let data = await manageProducts({}, 'read') -// data.result.map((product,i) => { -// const url = productCommonUrl(product.link); -// data.result[i].link = url -// }) -// fs.writeFileSync('updated.json', JSON.stringify(data.result)); -// const data = fs.readFileSync('updated.json').toString(); -// let products = JSON.parse(data); -// let arr = []; -// const duplicate = products.filter(product => { -// const same = products.filter(p => { -// return p.link === product.link -// }) -// const users = same.map((p) => ({ userId: p.userId, tracking_id: p.tracking_id})); -// arr.push({ -// link: product.link, -// merchant: product.merchant, -// initPrice: product.initPrice, -// price: product.price, -// title: product.title, -// users - -// }) -// // same.map((p, index) => { -// // delete products[index] -// // }) -// }) -// fs.writeFileSync('duplicate.json', JSON.stringify(arr)); - -// (async () => await func())() -const deleteData = async () => { - await connectDb(); - const db = mongo.db('TESTS').collection('tasks') - await db.deleteOne({users: null}) -} -(async () => await deleteData())() \ No newline at end of file From e25ffc851a9dfd0c85a73f88be77697d0caf17ba Mon Sep 17 00:00:00 2001 From: Siddiqui Affan Date: Tue, 20 Sep 2022 15:23:05 +0530 Subject: [PATCH 15/34] Create FUNDING.yml --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..9e51608 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +custom: ['https://paypal.me/affanthebest'] From a557ff014b1b62de72bd95a1bc82c5a587c67a02 Mon Sep 17 00:00:00 2001 From: Siddiqui Affan Date: Thu, 29 Sep 2022 11:49:49 +0000 Subject: [PATCH 16/34] add error handler --- bot.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot.js b/bot.js index f8bd0fb..050e67d 100644 --- a/bot.js +++ b/bot.js @@ -246,7 +246,7 @@ const track = async () => { [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] : [] } - }); + }).catch(e => {}) })) } catch (e) { bot.start() } } From f22bbb826e8bdc2d7895cc9de6257335fc6592cd Mon Sep 17 00:00:00 2001 From: ZAMEEL HASSAN Date: Thu, 24 Nov 2022 21:43:33 +0530 Subject: [PATCH 17/34] README update --- README.md | 50 ++++++++++++++++++-------------------------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 60abab5..a5555c0 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,34 @@ Logo

-# price-tracker [@AsTracker](https://t.me/AsPriceTrackerBot) +# Price Tracker -A Telegram bot that can track price of amazon & flipkart products (Soon more). -If you like this project, please leave a 🌟. +A Telegram bot that can track price of Amazon & flipkart products (more coming soon) ---- +![Star](https://img.shields.io/github/stars/siddiquiaffan/price-tracker?label=Star&logo=Github) +![GitHub Follow](https://img.shields.io/github/followers/siddiquiaffan?label=Follow&logo=GitHub) +![State](https://img.shields.io/github/deployments/siddiquiaffan/price-tracker/github-pages?color=blue) -- [Features](#features) -- [Usage](#how-to-use) -- [Deployment](#deploy) -- [Contributing](#contributing) -- --- ## Features -``` - Track Amazon Product. - Track Flipkart Product. - Notify on every price change. - Broadcast (Admin). - API support. -``` --- ## How to use -``` +> To use this bot in your Telegram, [click here](t.me/AsPriceTrackerBot) + /start - Start the bot /help - get this message. /track {Product Link} - Add product to tracking list. /stop {Tracking ID} - Stop tracking. /list - Get list of products that are being tracked. -``` --- @@ -42,42 +36,34 @@ If you like this project, please leave a 🌟. - [![Deploy with Heroku](https://www.herokucdn.com/deploy/button.svg "Deploy with Heroku")](https://heroku.com/deploy?template=https://github.com/siddiquiaffan/price-tracker "Deploy with Heroku") -- [![Deploy+on+Railway](https://railway.app/button.svg)](https://railway.app/new/template?template=https://github.com/siddiquiaffan/price-tracker&envs=ADMINS,BOT_TOKEN,DB_URL,WORKER_URL,API_KEY,LIMIT&ADMINSDesc=Telegarm+ids+of+admins+separated+by+space&BOT_TOKENDesc=Get+Your+Bot+Token+From+@BotFather.&DB_URLDesc=Create+A+Database+In+Mongodb+And+Get+URL.&WORKER_URLDesc=Paste+worker.js+code+in+Cloudfare+Worker+and+get+url.&API_KEYDesc=Any+secret+key+to+access+API&LIMITDesc=Limit+of+products+to+track+per+user.) +- [![Deploy on Railway](https://railway.app/button.svg "Deploy on Railway")](https://railway.app/new/template?template=https://github.com/siddiquiaffan/price-tracker&envs=ADMINS,BOT_TOKEN,DB_URL,WORKER_URL,API_KEY,LIMIT&ADMINSDesc=Telegarm+ids+of+admins+separated+by+space&BOT_TOKENDesc=Get+Your+Bot+Token+From+@BotFather.&DB_URLDesc=Create+A+Database+In+Mongodb+And+Get+URL.&WORKER_URLDesc=Paste+worker.js+code+in+Cloudfare+Worker+and+get+url.&API_KEYDesc=Any+secret+key+to+access+API&LIMITDesc=Limit+of+products+to+track+per+user. "Deploy on Railway") -- ### Local - +- ### Local - ``` - Clone Repository - Install Dependencies (npm install) - Create .env file and fill it with your details. - Start App (npm start) - ``` + --- ## Contributing -``` -- Fork this repo. -- Make changes. -- Create a pull request. -``` +- Fork this repo ![fork](https://img.shields.io/github/forks/siddiquiaffan/price-tracker?label=fork&logo=Github) +- Add your changes +- Create a pull request --- ## NOTE -``` -- Do not clone this repo (You can fork it instead). -- Use this bot at your own risk. -- This bot can be a little bit slow cuz using cloudfare workers for scrapping product data (To avoid ip blocking from Amazon & Flipkart) -``` +- Do not clone this repo (You can fork it instead) +- Use this bot at your own risk +- This bot can be a little bit slow because of using Cloudflare workers for scraping product data
+(To avoid IP blocking from Amazon & Flipkart) --- ## License -[LICENSE](https://github.com/siddiquiaffan/price-tracker/blob/main/LICENSE) - ---- - -### [Follow me on GitHub](https://github.com/siddiquiaffan) +[LICENSE](https://github.com/siddiquiaffan/price-tracker/blob/main/LICENSE) \ No newline at end of file From 6d678110a66a12973f3666bd5e615ff88c382280 Mon Sep 17 00:00:00 2001 From: ZAMEEL HASSAN Date: Thu, 24 Nov 2022 21:44:34 +0530 Subject: [PATCH 18/34] README update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a5555c0..4efec5a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ Logo

-# Price Tracker +# [Price Tracker](https://t.me/AsPriceTrackerBot) A Telegram bot that can track price of Amazon & flipkart products (more coming soon) From 1bf9b010d437894de4aa8346ef362dffc885b152 Mon Sep 17 00:00:00 2001 From: ZAMEEL HASSAN Date: Thu, 24 Nov 2022 21:45:32 +0530 Subject: [PATCH 19/34] README update --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4efec5a..fadd236 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@ A Telegram bot that can track price of Amazon & flipkart products (more coming s > To use this bot in your Telegram, [click here](t.me/AsPriceTrackerBot) -/start - Start the bot -/help - get this message. -/track {Product Link} - Add product to tracking list. -/stop {Tracking ID} - Stop tracking. -/list - Get list of products that are being tracked. +/start - Start the bot
+/help - get this message
+/track {Product Link} - Add product to tracking list
+/stop {Tracking ID} - Stop tracking
+/list - Get list of products that are being tracked
--- From fe2261c2a8441f953fe985cf965cd9dc9c5c3595 Mon Sep 17 00:00:00 2001 From: ZAMEEL HASSAN Date: Thu, 24 Nov 2022 21:47:01 +0530 Subject: [PATCH 20/34] README update --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fadd236..d6768c1 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,11 @@ A Telegram bot that can track price of Amazon & flipkart products (more coming s ## Deploy -- [![Deploy with Heroku](https://www.herokucdn.com/deploy/button.svg "Deploy with Heroku")](https://heroku.com/deploy?template=https://github.com/siddiquiaffan/price-tracker "Deploy with Heroku") +[![Deploy with Heroku](https://www.herokucdn.com/deploy/button.svg "Deploy with Heroku")](https://heroku.com/deploy?template=https://github.com/siddiquiaffan/price-tracker "Deploy with Heroku") -- [![Deploy on Railway](https://railway.app/button.svg "Deploy on Railway")](https://railway.app/new/template?template=https://github.com/siddiquiaffan/price-tracker&envs=ADMINS,BOT_TOKEN,DB_URL,WORKER_URL,API_KEY,LIMIT&ADMINSDesc=Telegarm+ids+of+admins+separated+by+space&BOT_TOKENDesc=Get+Your+Bot+Token+From+@BotFather.&DB_URLDesc=Create+A+Database+In+Mongodb+And+Get+URL.&WORKER_URLDesc=Paste+worker.js+code+in+Cloudfare+Worker+and+get+url.&API_KEYDesc=Any+secret+key+to+access+API&LIMITDesc=Limit+of+products+to+track+per+user. "Deploy on Railway") +[![Deploy on Railway](https://railway.app/button.svg "Deploy on Railway")](https://railway.app/new/template?template=https://github.com/siddiquiaffan/price-tracker&envs=ADMINS,BOT_TOKEN,DB_URL,WORKER_URL,API_KEY,LIMIT&ADMINSDesc=Telegarm+ids+of+admins+separated+by+space&BOT_TOKENDesc=Get+Your+Bot+Token+From+@BotFather.&DB_URLDesc=Create+A+Database+In+Mongodb+And+Get+URL.&WORKER_URLDesc=Paste+worker.js+code+in+Cloudfare+Worker+and+get+url.&API_KEYDesc=Any+secret+key+to+access+API&LIMITDesc=Limit+of+products+to+track+per+user. "Deploy on Railway") -- ### Local +Deploy locally: - Clone Repository - Install Dependencies (npm install) From 33a88a7beabd017d492e3a72a7d93a294071f1ba Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Wed, 22 Feb 2023 19:19:40 +0530 Subject: [PATCH 21/34] Porcess only 10 products at a time --- bot.js | 95 ++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/bot.js b/bot.js index 050e67d..fbe7785 100644 --- a/bot.js +++ b/bot.js @@ -52,17 +52,17 @@ const processUrl = async (msg, ctx) => { ctx.chat.id, sentMsg.message_id, `Sorry, I couldn't track this product. Make sure you've sent correct product link.`, { parse_mode: "Markdown", reply_markup } - ); + ).catch(e => {}); } } else { ctx.reply( "I'm sorry, but you can't add more products as you've already reached the maximum limit.\n\nPlease delete atleast one product. And try again.\n\nTo get list send /list", - { reply_to_message_id: ctx.message.message_id } ); + { reply_to_message_id: ctx.message.message_id } ).catch(e => {}); } } else { - ctx.reply( `Sorry, I can't track this product. Cuz the link you sent is not a amazon or flipkart product link.` ); + ctx.reply( `Sorry, I can't track this product. Cuz the link you sent is not a amazon or flipkart product link.` ).catch(e => {}); } } else { - ctx.reply( `Sorry ${ctx.message.chat.first_name}, I can't track this product. Make sure you've sent correct product link.` ); + ctx.reply( `Sorry ${ctx.message.chat.first_name}, I can't track this product. Make sure you've sent correct product link.` ).catch(e => {}); } } catch(e){ console.error(e) @@ -76,7 +76,7 @@ bot.command("start", (ctx) => { ctx.reply( `Hello ${ctx.message.chat.first_name}, I can track price for Amazon & Flipkart products (Soon more).\n\nCheck /help to get started.\n`, { reply_to_message_id: ctx.message.message_id, reply_markup, } - ); + ).catch(() => {}) manageUsers( { id: ctx.message.from.id, name: ctx.message.from.first_name }, "update" ); } catch (e) { console.log("Error", e); @@ -92,7 +92,7 @@ bot.command("help", (ctx) => { reply_to_message_id: ctx.message.message_id, reply_markup, } - ); + ).catch(() => {}) } catch (e) { } }); @@ -224,37 +224,60 @@ bot.callbackQuery("stopTracking", async (ctx) => { const track = async () => { try { const products = await manageProducts({}, "read"); - await Promise.all( - products.result.map(async (product) => { - const details = await getProductDetails(product.link, product.merchant); - if (details.ok && !isNaN(details.price) && details.price !== product.price) { - try { - await manageProducts({ tracking_id: product.tracking_id, userId: product.userId, merchant: product.merchant, title: details.title, link: product.link, initPrice: product.price, price: details.price, users: product.users}, "update"); - await Promise.all(product.users.map(async user => { - bot.api.sendMessage( - user.userId, - ` Price has been ${details.price > product.price ? "increased" : "decreased" - } by ${Math.abs(product.price - details.price)}. \n\n${details.title - }\n\nCurrent Price: ${details.price}\nLink: ${product.merchant}\n\nTo stop tracking send /stop_${user.tracking_id - }`, - { - parse_mode: "HTML", - reply_markup: { - inline_keyboard: details?.link ? [ - [{ text: "Buy Now", url: details.link }], - [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] - : [] - } - }).catch(e => {}) - })) - } catch (e) { bot.start() } - } - }) - ); - } catch (e) { } + // Process 10 products at a time + for (let i = 0; i < products.result.length; i = i + 10 ) { + const temp = products.result.slice(i, i + 10) + + await Promise.all( + temp.map(async (product) => { + const details = await getProductDetails(product.link, product.merchant); + + if (details.ok && !isNaN(details.price) && details.price !== product.price) { + try { + await manageProducts({ tracking_id: product.tracking_id, userId: product.userId, merchant: product.merchant, title: details.title, link: product.link, initPrice: product.price, price: details.price, users: product.users}, "update"); + await Promise.all(product.users.map(async user => { + bot.api.sendMessage( + user.userId, + ` Price has been ${details.price > product.price ? "increased" : "decreased" + } by ${Math.abs(product.price - details.price)}. \n\n${details.title + }\n\nCurrent Price: ${details.price}\nLink: ${product.merchant}\n\nTo stop tracking send /stop_${user.tracking_id + }`, + { + parse_mode: "HTML", + reply_markup: { + inline_keyboard: details?.link ? [ + [{ text: "Buy Now", url: details.link }], + [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] + : [] + } + }).catch(e => console.log(`🚀 ~ file: bot.js:255 ~ temp.map ~ e:`, e)) + })) + + // wait for 1 sec + await new Promise(resolve => setTimeout(resolve, 1000)) + } catch (e) { + console.log(`🚀 ~ file: bot.js:260 ~ temp.map ~ e:`, e) + bot.start() + // wait for 5 sec + await new Promise(resolve => setTimeout(resolve, 5000)) + } + } + }) + ); + } + } catch (e) { + console.log(`🚀 ~ file: bot.js:270 ~ track ~ e:`, e) + } }; +bot.command("update", async (ctx) => { + if (ADMINS.includes(ctx.from.id)) { + track() + ctx.reply("Updating products...") + } +}) + bot.catch((err) => { console.error("err"); const ctx = err.ctx; @@ -266,4 +289,4 @@ bot.catch((err) => { setInterval(track, 3600000); //Track every hr. -export default bot; +export default bot \ No newline at end of file From e83ee48c348538c648083653c5015b7907f29b9e Mon Sep 17 00:00:00 2001 From: Siddiqui Affan Date: Wed, 22 Feb 2023 22:31:53 +0530 Subject: [PATCH 22/34] Use ParseFloat --- utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils.js b/utils.js index 46a5fc8..02a1d44 100644 --- a/utils.js +++ b/utils.js @@ -61,7 +61,7 @@ const getProductDetails = async(url, merchant) => { }); const $ = cheerio.load(res.data); const selector = selectors[merchant]; - const price = parseInt($(selector.price1).text().trim().replace(/^\D+|[^0-9.]/g, '')) || parseInt($(selector.price2).text().trim().replace(/^\D+|[^0-9.]/g, '')); + const price = parseFloat($(selector.price1).text().trim().replace(/^\D+|[^0-9.]/g, '')) || parseFloat($(selector.price2).text().trim().replace(/^\D+|[^0-9.]/g, '')); const title = $(selector.title).text().trim(); const image = $(selector.image1).attr('src'); if(!title || !price) { @@ -74,4 +74,4 @@ const getProductDetails = async(url, merchant) => { } } -export { isUrl, getRandomId, getProductDetails, productCommonUrl }; \ No newline at end of file +export { isUrl, getRandomId, getProductDetails, productCommonUrl }; From 4b64f16bf0a770d8e660d2f49a5c36be8ce9e58c Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Mon, 27 Mar 2023 15:27:21 +0530 Subject: [PATCH 23/34] add .dockerignore --- .dockerignore | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c83032b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +# flyctl launch added from .gitignore +node_modules +package-lock.json +logs +_IGNORE_session + +# managed +data.json +node-persis +qr.png +# end managed + +# managed +_IGNORE_ +WWebJS +# end managed + +temp + +#git +**/.git \ No newline at end of file From 6dfccae3fc6164d638ddbbff9572c9788a898fd8 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Mon, 27 Mar 2023 15:27:37 +0530 Subject: [PATCH 24/34] ignore fly.io config --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 96bc57e..57967b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# fly.io +fly.toml + + #Tests test.json # Logs From a7fbb93cd27622f27e87ea923e3e09aa480ac2e9 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Mon, 27 Mar 2023 15:28:04 +0530 Subject: [PATCH 25/34] Add Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0323c27..96f7172 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16 +FROM node:16-alpine # Create app directory WORKDIR / From da50309813caee9b7360ca8717ba55f7063b1e0d Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Mon, 27 Mar 2023 15:28:16 +0530 Subject: [PATCH 26/34] Removed default tag --- bot.js | 11 +++++------ utils.js | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/bot.js b/bot.js index fbe7785..99c1a0e 100644 --- a/bot.js +++ b/bot.js @@ -1,7 +1,7 @@ // Imports import { Bot } from "grammy" import { BOT_TOKEN, ADMINS, LIMIT } from "./config.js" -import { isUrl, getRandomId, getProductDetails } from "./utils.js" +import { isUrl, getRandomId, getProductDetails, productCommonUrl } from "./utils.js" import { manageProducts, manageUsers } from "./db.js" import unshort from "./unshort.js" @@ -28,12 +28,11 @@ const processUrl = async (msg, ctx) => { if (isUrl(productUrl)) { const merchant = productUrl.replace("www.", "").split("//")[1].split(".")[0]; if (merchant.match(/amazon|flipkart|snapdeal/gi)) { - const noOfProducts = ( - await manageProducts({ userId: ctx.from.id }, "read") - )?.result?.length; + const noOfProducts = ( await manageProducts({ userId: ctx.from.id }, "read") )?.result?.length; if (noOfProducts < LIMIT) { const sentMsg = await ctx.reply(`Tracking ${merchant} product...`, { reply_to_message_id: ctx.message.message_id }); const details = await getProductDetails(productUrl, merchant); + if (details.ok) { try { const tracking_id = getRandomId(); @@ -240,14 +239,14 @@ const track = async () => { user.userId, ` Price has been ${details.price > product.price ? "increased" : "decreased" } by ${Math.abs(product.price - details.price)}. \n\n${details.title - }\n\nCurrent Price: ${details.price}\nLink: ${product.merchant}\n\nTo stop tracking send /stop_${user.tracking_id }`, { parse_mode: "HTML", reply_markup: { inline_keyboard: details?.link ? [ - [{ text: "Buy Now", url: details.link }], + [{ text: "Buy Now", url: productCommonUrl(details.link, true) }], [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] : [] } diff --git a/utils.js b/utils.js index 46a5fc8..e2318a9 100644 --- a/utils.js +++ b/utils.js @@ -1,4 +1,4 @@ -import cheerio from 'cheerio' +import * as cheerio from 'cheerio' import axios from 'axios' import {WORKER_URL} from './config.js' @@ -28,7 +28,7 @@ const selectors = { } } -const productCommonUrl = (link) => { +const productCommonUrl = (link, tag) => { const url = new URL(link?.replace("www.", "")); const merchant = url.hostname.split(".")[0]; let id, commonUrl; @@ -37,7 +37,7 @@ const productCommonUrl = (link) => { id = link.match( /https?:\/\/(www\.)?(.*)amazon\.([a-z\.]{2,6})(\/d\/(.*)|\/(.*)\/?(?:dp|o|gp|-)\/)(aw\/d\/|product\/)?(B[0-9]{1}[0-9A-Z]{8}|[0-9]{9}(?:X|[0-9]))/i ).splice(-1)[0]; - commonUrl = "https://www.amazon.in/dp/" + id + "?tag=asloot-21"; + commonUrl = "https://www.amazon.in/dp/" + id + `${tag ? ('?tag=' + 'asloot-21') : ''}`; break; case "flipkart": id = url.searchParams.get("pid"); From 9eda82cc8fdf68040386235a8b6b0a8fbfa40fd5 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 22:22:54 +0530 Subject: [PATCH 27/34] Prettify --- bot.js | 115 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/bot.js b/bot.js index 99c1a0e..c2a35bb 100644 --- a/bot.js +++ b/bot.js @@ -23,27 +23,31 @@ const reply_markup = { const processUrl = async (msg, ctx) => { try { - const url = await unshort(msg); - const productUrl = "http" + url.split("http")[1].split(" ")[0].replace("dl.", "www.") - if (isUrl(productUrl)) { - const merchant = productUrl.replace("www.", "").split("//")[1].split(".")[0]; - if (merchant.match(/amazon|flipkart|snapdeal/gi)) { - const noOfProducts = ( await manageProducts({ userId: ctx.from.id }, "read") )?.result?.length; + const url = await unshort(msg); + const productUrl = "http" + url.split("http")[1].split(" ")[0].replace("dl.", "www.") + + if (isUrl(productUrl)) { + const merchant = productUrl.replace("www.", "").split("//")[1].split(".")[0]; + + if (merchant.match(/amazon|flipkart|snapdeal/gi)) { + const noOfProducts = (await manageProducts({ userId: ctx.from.id }, "read"))?.result?.length; + if (noOfProducts < LIMIT) { const sentMsg = await ctx.reply(`Tracking ${merchant} product...`, { reply_to_message_id: ctx.message.message_id }); const details = await getProductDetails(productUrl, merchant); - + if (details.ok) { try { const tracking_id = getRandomId(); await manageProducts( { tracking_id, userId: ctx.from.id, merchant, title: details.title, link: details.link, initPrice: details.price, price: details.price, }, "update" - ); - await ctx.api.editMessageText( - ctx.chat.id, sentMsg.message_id, - ` \nTracking ${details.title}\n\nCurrent Price: ${details.price}\nLink: ${merchant}\n\nTo stop tracking send /stop_${tracking_id}`, - { parse_mode: "HTML", reply_markup } + ); + + await ctx.api.editMessageText( + ctx.chat.id, sentMsg.message_id, + ` \nTracking ${details.title}\n\nCurrent Price: ${details.price}\nLink: ${merchant}\n\nTo stop tracking send /stop_${tracking_id}`, + { parse_mode: "HTML", reply_markup } ); } catch (e) { } } else { @@ -51,21 +55,23 @@ const processUrl = async (msg, ctx) => { ctx.chat.id, sentMsg.message_id, `Sorry, I couldn't track this product. Make sure you've sent correct product link.`, { parse_mode: "Markdown", reply_markup } - ).catch(e => {}); - } - } else { - ctx.reply( "I'm sorry, but you can't add more products as you've already reached the maximum limit.\n\nPlease delete atleast one product. And try again.\n\nTo get list send /list", - { reply_to_message_id: ctx.message.message_id } ).catch(e => {}); + ).catch(e => { }); } + } else { - ctx.reply( `Sorry, I can't track this product. Cuz the link you sent is not a amazon or flipkart product link.` ).catch(e => {}); + ctx.reply("I'm sorry, but you can't add more products as you've already reached the maximum limit.\n\nPlease delete atleast one product. And try again.\n\nTo get list send /list", + { reply_to_message_id: ctx.message.message_id }).catch(e => { }); } + } else { - ctx.reply( `Sorry ${ctx.message.chat.first_name}, I can't track this product. Make sure you've sent correct product link.` ).catch(e => {}); + ctx.reply(`Sorry, I can't track this product. Cuz the link you sent is not a amazon or flipkart product link.`).catch(e => { }); } - } catch(e){ - console.error(e) + } else { + ctx.reply(`Sorry ${ctx.message.chat.first_name}, I can't track this product. Make sure you've sent correct product link.`).catch(e => { }); } + } catch (e) { + console.error(e) + } } @@ -74,9 +80,13 @@ bot.command("start", (ctx) => { try { ctx.reply( `Hello ${ctx.message.chat.first_name}, I can track price for Amazon & Flipkart products (Soon more).\n\nCheck /help to get started.\n`, - { reply_to_message_id: ctx.message.message_id, reply_markup, } - ).catch(() => {}) - manageUsers( { id: ctx.message.from.id, name: ctx.message.from.first_name }, "update" ); + { + reply_to_message_id: ctx.message.message_id, + reply_markup, + } + ).catch(() => { }) + + manageUsers({ id: ctx.message.from.id, name: ctx.message.from.first_name }, "update"); } catch (e) { console.log("Error", e); } @@ -91,7 +101,7 @@ bot.command("help", (ctx) => { reply_to_message_id: ctx.message.message_id, reply_markup, } - ).catch(() => {}) + ).catch(() => { }) } catch (e) { } }); @@ -100,16 +110,18 @@ bot.command("track", async (ctx) => { const message = ctx.message.text.replace("/track ", ""); processUrl(message, ctx); }); - + bot.command("list", async (ctx) => { try { const products = await manageProducts({ 'users.userId': ctx.from.id }, "read"); + const list = products.result .map( (product) => `${product.title}\nLast Price: ${product.price}\nLink: ${product.merchant}\nTo stop send /stop_${product.users.filter(u => u.userId == ctx.from.id)[0].tracking_id}` ) .join("\n\n"); + ctx.reply(`Here is your tracking list:\n\n${list}`, { reply_to_message_id: ctx.message.message_id, parse_mode: "HTML", @@ -124,10 +136,12 @@ bot.command("list", async (ctx) => { bot.hears(/^\/stop_([a-z0-9])/, async (ctx) => { const tracking_id = ctx.message.text.replace("/stop_", ""); + const result = await manageProducts( { tracking_id, userId: ctx.from.id }, "delete" ); + ctx.reply( result.ok ? `Stopped tracking product with tracking id ${tracking_id}` @@ -139,7 +153,9 @@ bot.command("broadcast", async (ctx) => { if (ADMINS.includes(ctx.from.id)) { let msg = ctx.message.text.replace("/broadcast ", ""); const inline_keyboard = ctx.message.text.split("inline_keyboard:")[1]; + msg = msg.replace("inline_keyboard:", "").replace(inline_keyboard, ""); + const users = await manageUsers({}, "read"); await Promise.all( users.result.map(async (user) => { @@ -168,6 +184,7 @@ bot.command("broadcast", async (ctx) => { bot.command("users", async (ctx) => { if (ADMINS.includes(ctx.from.id)) { let users = await manageUsers({}, "read"); + users = "List Of Users: \n\n" + users.result @@ -176,27 +193,30 @@ bot.command("users", async (ctx) => { `${user.id} - ${user.name}` ) .join("\n"); + ctx.reply(users, { parse_mode: "HTML" }); } }); bot.command("stats", async (ctx) => { - try{ - const[users, products] = await Promise.all([manageUsers, manageProducts].map( + try { + const [users, products] = await Promise.all([manageUsers, manageProducts].map( async (func) => await func({}, "read") + )); let prodCount = 0; products.result.map(prod => prodCount += prod.users.length); + ctx.reply( `Total Users: ${users.result.length}\nTotal Products: ${prodCount}` ); - }catch(e){ + } catch (e) { console.log(e) } }); bot.on('::url', async ctx => { - if(ctx.chat.type === "private"){ + if (ctx.chat.type === "private") { const message = ctx.message.text; processUrl(message, ctx); } @@ -207,10 +227,12 @@ bot.callbackQuery("stopTracking", async (ctx) => { ctx.update?.callback_query?.message?.reply_markup?.inline_keyboard[1][0]?.text?.split( " - " )[1]; + const result = await manageProducts( { tracking_id, userId: ctx.from.id }, "delete" ); + ctx.api.editMessageText( ctx.update?.callback_query?.message?.chat?.id, ctx.update?.callback_query?.message?.message_id, @@ -224,16 +246,26 @@ const track = async () => { try { const products = await manageProducts({}, "read"); // Process 10 products at a time - for (let i = 0; i < products.result.length; i = i + 10 ) { - const temp = products.result.slice(i, i + 10) - + for (let i = 0; i < products.result.length; i = i + 10) { + const temp = products.result.slice(i, i + 10) + await Promise.all( temp.map(async (product) => { const details = await getProductDetails(product.link, product.merchant); if (details.ok && !isNaN(details.price) && details.price !== product.price) { try { - await manageProducts({ tracking_id: product.tracking_id, userId: product.userId, merchant: product.merchant, title: details.title, link: product.link, initPrice: product.price, price: details.price, users: product.users}, "update"); + await manageProducts({ + tracking_id: product.tracking_id, + userId: product.userId, + merchant: product.merchant, + title: details.title, + link: product.link, + initPrice: product.price, + price: details.price, + users: product.users + }, "update"); + await Promise.all(product.users.map(async user => { bot.api.sendMessage( user.userId, @@ -246,28 +278,28 @@ const track = async () => { parse_mode: "HTML", reply_markup: { inline_keyboard: details?.link ? [ - [{ text: "Buy Now", url: productCommonUrl(details.link, true) }], - [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] - : [] + [{ text: "Buy Now", url: productCommonUrl(details.link, true) }], + [{ text: "Stop Tracking - " + user.tracking_id, callback_data: `stopTracking`, }]] + : [] } }).catch(e => console.log(`🚀 ~ file: bot.js:255 ~ temp.map ~ e:`, e)) })) // wait for 1 sec await new Promise(resolve => setTimeout(resolve, 1000)) - } catch (e) { + } catch (e) { console.log(`🚀 ~ file: bot.js:260 ~ temp.map ~ e:`, e) bot.start() // wait for 5 sec await new Promise(resolve => setTimeout(resolve, 5000)) - } + } } }) ); } } catch (e) { console.log(`🚀 ~ file: bot.js:270 ~ track ~ e:`, e) - } + } }; bot.command("update", async (ctx) => { @@ -279,10 +311,13 @@ bot.command("update", async (ctx) => { bot.catch((err) => { console.error("err"); + const ctx = err.ctx; console.error(`Error while handling update ${ctx.update.update_id}:`); + const e = err.error; console.error("Error: ", e.description); + bot.start(); }); From f4d2f995be547cd69390cdccc5a6f091eb914d7e Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 22:24:53 +0530 Subject: [PATCH 28/34] Add HTTPS Proxy URL --- config.js | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/config.js b/config.js index bead029..71503ef 100644 --- a/config.js +++ b/config.js @@ -11,11 +11,25 @@ if(!process.env.DB_URL) { process.exit(1); } -const ADMINS = process.env.ADMINS || '' -const BOT_TOKEN = process.env.BOT_TOKEN || '' -const DB_URL = process.env.DB_URL || '' -const WORKER_URL = process.env.WORKER_URL || '' -const API_KEY = process.env.API_KEY || '' // Generate any API Key and pass it when accessing the API. +/** List of amdins (tg IDs) separated by space */ +const ADMINS = process.env.ADMINS ?? '' + +/** Telegram bot token */ +const BOT_TOKEN = process.env.BOT_TOKEN ?? '' + +/** HTTPS Proxy URL */ +const HTTPS_PROXY = process.env.PROXY ?? '' + +/** MongoDB URL */ +const DB_URL = process.env.DB_URL ?? '' + +/** Cloudflare Worker URL */ +const WORKER_URL = process.env.WORKER_URL ?? '' + +/** API Key - A random secure key to access api */ +const API_KEY = process.env.API_KEY ?? '' // Generate any API Key and pass it when accessing the API. + +/** Maximum number of products can be added by a user at a time. */ const LIMIT = Number(process.env.LIMIT) // Maximum number of products can be added by a user at a time. - -export { ADMINS, BOT_TOKEN, DB_URL, WORKER_URL, API_KEY, LIMIT } \ No newline at end of file + +export { ADMINS, BOT_TOKEN, DB_URL, WORKER_URL, API_KEY, LIMIT, HTTPS_PROXY } \ No newline at end of file From e4af450c45502821f9cfb8d5cad30efcf06f0f70 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 22:25:08 +0530 Subject: [PATCH 29/34] Install https-proxy-agent --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 2e3bdbc..4342899 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "dotenv": "^10.0.0", "express": "^4.17.1", "grammy": "^1.5.4", + "https-proxy-agent": "^7.0.2", "mongodb": "^4.9.0" } } From 9b9a282e3c9725371b0df44b17cc412def5b625b Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 22:25:50 +0530 Subject: [PATCH 30/34] Return URL as it is if it's already unshorted --- unshort.js | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/unshort.js b/unshort.js index ca65765..b6b163c 100644 --- a/unshort.js +++ b/unshort.js @@ -1,19 +1,48 @@ import axios from 'axios'; + +const merchants = [ + 'amazon', + 'flipkart', + 'snapdeal', + 'ajio', + 'myntra', + 'jabong', + 'bewakoof', + 'limeroad', + 'shein', +] + +/** + * Get the long url from a short url + * @param {string} url + * @returns + */ const unshort = async (url) => { const extractUrl = req => req?.request?.res?.responseUrl || req?.request?._redirectable?._currentUrl || req?.request?._currentUrl || req?.request?._options?.href || 'https://' + req?.request?.host + req?.request?.path; + + + const host = new URL(url).hostname.split("."); + const merchant = host[0] === 'www' ? host[1] : host[0]; + + if (merchants.includes(merchant)) + return url; + + let longUrl = url; + try { const req = await axios.get(url); const result = extractUrl(req); - var longUrl = result ? result : url; + longUrl = result ? result : url; } catch (err) { const result = extractUrl(err); - var longUrl = result ? result : url; + longUrl = result ? result : url; } return longUrl; } + export default unshort; \ No newline at end of file From 0542f40f302142273706b5a5401f802807d247e1 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 22:26:15 +0530 Subject: [PATCH 31/34] Use proxy, update selectors --- utils.js | 139 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 98 insertions(+), 41 deletions(-) diff --git a/utils.js b/utils.js index ac3d53e..f377ebd 100644 --- a/utils.js +++ b/utils.js @@ -1,33 +1,51 @@ import * as cheerio from 'cheerio' import axios from 'axios' -import {WORKER_URL} from './config.js' +import { WORKER_URL, HTTPS_PROXY } from './config.js' +import { HttpsProxyAgent } from 'https-proxy-agent'; const urlRegex = /^(?:(?:(?:https?|ftp):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:[/?#]\S*)?$/i; + +/** + * check if string is url or not + * @param {string} str + * @returns {boolean} + */ const isUrl = (str) => { // Return true if string is a valid URL - return urlRegex.test(str); + return urlRegex.test(str); }; -const getRandomId = () => [...Array(10)].map(i=>(~~(Math.random()*36)).toString(36)).join(''); // Return a random id +/** + * Generate random string id + * @returns {string} + */ +const getRandomId = () => [...Array(10)].map(i => (~~(Math.random() * 36)).toString(36)).join(''); + const selectors = { - amazon: { - title: '#productTitle', - price1: 'span.a-price.a-text-price.a-size-medium.apexPriceToPay > span:nth-child(2)', price2: 'span.a-price.aok-align-center.priceToPay > span.a-offscreen', - image1: '#landingImage' - }, - flipkart: { - title: '.B_NuCI', - price1: '._30jeq3._16Jk6d', - image1: '#container > div > div._2c7YLP.UtUXW0._6t1WkM._3HqJxg > div._1YokD2._2GoDe3 > div._1YokD2._3Mn1Gg.col-5-12._78xt5Y > div:nth-child(1) > div > div._3li7GG > div._1BweB8 > div._3kidJX > div.CXW8mj._3nMexc > img', - }, - snapdeal: { - title: '#productOverview > div.col-xs-14.right-card-zoom.reset-padding > div > div.pdp-fash-topcenter-inner.layout > div.row > div.col-xs-18 > h1', - price1: '#buyPriceBox > div.row.reset-margin > div.col-xs-14.reset-padding.padL8 > div.disp-table > div.pdp-e-i-PAY-r.disp-table-cell.lfloat > span.pdp-final-price > span', - image1: '#bx-slider-left-image-panel > li:nth-child(1) > img' - } + amazon: { + title: '#productTitle', + // price1: 'span.a-price.a-text-price.a-size-medium.apexPriceToPay > span:nth-child(2)', price2: 'span.a-price.aok-align-center.priceToPay > span.a-offscreen', + price1: '#tp_price_block_total_price_ww span', + image1: '#landingImage' + }, + flipkart: { + title: '.B_NuCI', + price1: '._30jeq3._16Jk6d', + image1: '#container > div > div._2c7YLP.UtUXW0._6t1WkM._3HqJxg > div._1YokD2._2GoDe3 > div._1YokD2._3Mn1Gg.col-5-12._78xt5Y > div:nth-child(1) > div > div._3li7GG > div._1BweB8 > div._3kidJX > div.CXW8mj._3nMexc > img', + }, + snapdeal: { + title: '#productOverview > div.col-xs-14.right-card-zoom.reset-padding > div > div.pdp-fash-topcenter-inner.layout > div.row > div.col-xs-18 > h1', + price1: '#buyPriceBox > div.row.reset-margin > div.col-xs-14.reset-padding.padL8 > div.disp-table > div.pdp-e-i-PAY-r.disp-table-cell.lfloat > span.pdp-final-price > span', + image1: '#bx-slider-left-image-panel > li:nth-child(1) > img' + } } +/** + * Get common url for the product + * @param {string} link + * @param {string} tag +*/ const productCommonUrl = (link, tag) => { const url = new URL(link?.replace("www.", "")); const merchant = url.hostname.split(".")[0]; @@ -41,7 +59,7 @@ const productCommonUrl = (link, tag) => { break; case "flipkart": id = url.searchParams.get("pid"); - commonUrl = id ? "https://www.flipkart.com/product/p/itme?pid=" + id : link.includes('/p/itm') ?link.split('?')[0] : link; + commonUrl = id ? "https://www.flipkart.com/product/p/itme?pid=" + id : link.includes('/p/itm') ? link.split('?')[0] : link; break; default: null; @@ -50,28 +68,67 @@ const productCommonUrl = (link, tag) => { return commonUrl; }; -const getProductDetails = async(url, merchant) => { - try{ - const commonUrl = productCommonUrl(url); - const res = await axios.get(`${WORKER_URL}/?url=${encodeURIComponent(commonUrl)}`, { - headers: { - "User-Agent": - "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36", - }, - }); - const $ = cheerio.load(res.data); - const selector = selectors[merchant]; - const price = parseFloat($(selector.price1).text().trim().replace(/^\D+|[^0-9.]/g, '')) || parseFloat($(selector.price2).text().trim().replace(/^\D+|[^0-9.]/g, '')); - const title = $(selector.title).text().trim(); - const image = $(selector.image1).attr('src'); - if(!title || !price) { - return {ok: false} - } - return {ok: true, title, price, image, link: commonUrl} - }catch(e){ - console.log(e); - return {ok: false} - } + +/** + * Make request to the url + * @param {string} url + * @param {object} options + */ +const makeRequest = async (url, { method = 'GET', useProxy }) => { + try { + + const options = { method, headers: { "User-Agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" } }; + if (useProxy) options.httpsAgent = new HttpsProxyAgent(HTTPS_PROXY); + + if (HTTPS_PROXY && useProxy) + return await axios(url, options); + + // else if (WORKER_URL) { + // const url1 = new URL(WORKER_URL) + // url1.searchParams.set('url', url) + + // return await axios(url1.toString(), options); + // } + + else return await axios(url, options); + } catch (err) { + throw err; + } +} + +/** + * Get product details from the url + * @param {string} url + * @param {string} merchant + * @returns + */ +const getProductDetails = async (url, merchant) => { + try { + const commonUrl = productCommonUrl(url); + const res = await makeRequest(commonUrl, { useProxy: commonUrl.includes('amazon.') }); + + const $ = cheerio.load(res.data); + const selector = selectors[merchant]; + + const priceEl = $(selector.price1) || $(selector.price2); + if (!priceEl || !priceEl.text()?.trim()) + return { ok: false } + + const price = priceEl.text()?.split('.')[0]?.trim().replace(/^\D+|[^0-9.]/g, ''); + + // const price = parseFloat($(selector.price1).text().trim().replace(/^\D+|[^0-9.]/g, '')) || parseFloat($(selector.price2).text().trim().replace(/^\D+|[^0-9.]/g, '')); + + const title = $(selector.title).text().trim(); + const image = $(selector.image1).attr('src'); + + if (!title || !price) + return { ok: false } + + return { ok: true, title, price, image, link: commonUrl } + } catch (e) { + console.log(e); + return { ok: false } + } } export { isUrl, getRandomId, getProductDetails, productCommonUrl }; From 6049f09343a19128f9dee9bc4c0ea9adc81f8590 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 23:06:39 +0530 Subject: [PATCH 32/34] Switch to webhook --- bot.js | 4 ++-- index.js | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/bot.js b/bot.js index c2a35bb..8ff7145 100644 --- a/bot.js +++ b/bot.js @@ -289,7 +289,7 @@ const track = async () => { await new Promise(resolve => setTimeout(resolve, 1000)) } catch (e) { console.log(`🚀 ~ file: bot.js:260 ~ temp.map ~ e:`, e) - bot.start() + // bot.start() // wait for 5 sec await new Promise(resolve => setTimeout(resolve, 5000)) } @@ -318,7 +318,7 @@ bot.catch((err) => { const e = err.error; console.error("Error: ", e.description); - bot.start(); + // bot.start(); }); setInterval(track, 3600000); //Track every hr. diff --git a/index.js b/index.js index 0c129dc..7490c34 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ import { manageProducts, manageUsers } from "./db.js" import {API_KEY} from './config.js' import express from 'express' import bot from './bot.js' +import { webhookCallback } from "grammy" //Globals const port = process.env.PORT || 3000; @@ -81,5 +82,21 @@ app.get('/info', async(req, res) => { res.send(JSON.stringify({error: 'Invalid API key'})) }) +// use bot webhook path +app.use('/bot', webhookCallback(bot, 'express')); + +// set bot webhook, use req.url as webhook path +app.get('/setup', async (req, res) => { + // get host name from req + try { + const host = req.hostname; + bot.api.setWebhook(`https://${host}/bot`); + res.send('ok'); + } catch (e) { + console.log(e); + res.send('error'); + }; +}) + app.listen(port, async () => console.log('listening to port ' + port)); -bot.start().then(() => console.log('Bot launched!')); \ No newline at end of file +// bot.start().then(() => console.log('Bot launched!')); \ No newline at end of file From 9abeeb319c2d14b108e09959532b56c14d014098 Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 23:14:26 +0530 Subject: [PATCH 33/34] Add setup docs --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index d6768c1..5383bc0 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ A Telegram bot that can track price of Amazon & flipkart products (more coming s ## Deploy +**NOTE** - We're now using https proxy for sending requests. So add your proxy url in env as `PROXY` + [![Deploy with Heroku](https://www.herokucdn.com/deploy/button.svg "Deploy with Heroku")](https://heroku.com/deploy?template=https://github.com/siddiquiaffan/price-tracker "Deploy with Heroku") [![Deploy on Railway](https://railway.app/button.svg "Deploy on Railway")](https://railway.app/new/template?template=https://github.com/siddiquiaffan/price-tracker&envs=ADMINS,BOT_TOKEN,DB_URL,WORKER_URL,API_KEY,LIMIT&ADMINSDesc=Telegarm+ids+of+admins+separated+by+space&BOT_TOKENDesc=Get+Your+Bot+Token+From+@BotFather.&DB_URLDesc=Create+A+Database+In+Mongodb+And+Get+URL.&WORKER_URLDesc=Paste+worker.js+code+in+Cloudfare+Worker+and+get+url.&API_KEYDesc=Any+secret+key+to+access+API&LIMITDesc=Limit+of+products+to+track+per+user. "Deploy on Railway") @@ -47,6 +49,14 @@ Deploy locally: --- +### POST-DEPLOYMENT +**Setup bot:** Get your deployment url and navigate to {YOUR_DEPLOYMENT_URL}/setup +``` +Example: https://price-tracker.herokuapp.com/setup +``` +=> Replace `https://price-tracker.herokuapp.com/` with your deployment url. + +--- ## Contributing - Fork this repo ![fork](https://img.shields.io/github/forks/siddiquiaffan/price-tracker?label=fork&logo=Github) From ae5939de447762885db018690fea3b3dfc1533aa Mon Sep 17 00:00:00 2001 From: siddiquiaffan Date: Sat, 7 Oct 2023 23:45:08 +0530 Subject: [PATCH 34/34] Use long polling in development mode --- index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 7490c34..a675612 100644 --- a/index.js +++ b/index.js @@ -99,4 +99,8 @@ app.get('/setup', async (req, res) => { }) app.listen(port, async () => console.log('listening to port ' + port)); -// bot.start().then(() => console.log('Bot launched!')); \ No newline at end of file + +// if NODE_ENV is development, start polling +if (process.env.NODE_ENV === 'development') { + bot.start(); +} \ No newline at end of file