This commit is contained in:
Lukian LEIZOUR 2022-11-19 01:49:12 +01:00
parent be4fd23bcf
commit 0bd53741af
728 changed files with 86573 additions and 0 deletions

6
node_modules/googlethis/.eslintignore generated vendored Normal file
View file

@ -0,0 +1,6 @@
.git
.github
images/
test/
coverage/
node_modules/

46
node_modules/googlethis/.eslintrc.yml generated vendored Normal file
View file

@ -0,0 +1,46 @@
env:
commonjs: true
es2021: true
node: true
extends: eslint:recommended
parserOptions:
ecmaVersion: latest
rules:
max-len:
- error
-
code: 200
ignoreComments: true
ignoreTrailingComments: true
ignoreStrings: true
ignoreTemplateLiterals: true
ignoreRegExpLiterals: true
quotes: [error, single]
no-template-curly-in-string: error
no-unreachable-loop: error
no-unused-private-class-members: error
getter-return: 'off'
no-prototype-builtins: 'off'
no-async-promise-executor: 'off'
no-case-declarations: 'off'
no-return-assign: 'off'
no-floating-decimal: error
no-implied-eval: error
arrow-spacing: error
no-invalid-this: error
no-lone-blocks: error
no-new-func: error
no-new-wrappers: error
no-new: error
no-void: error
no-octal-escape: error
no-self-compare: error
no-sequences: error
no-throw-literal: error
no-unmodified-loop-condition: error
no-useless-call: error
no-useless-concat: error
no-useless-escape: error
no-useless-return: error

View file

@ -0,0 +1 @@
blank_issues_enabled: true

View file

@ -0,0 +1,33 @@
name: Feature request
description: Use this template to suggest new features
labels: [enhancement]
body:
- type: textarea
id: feature-description
attributes:
label: Describe your suggestion
placeholder: How would it work?
validations:
required: true
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: checkboxes
id: checklist
attributes:
label: Checklist
options:
- label: I am running the latest version.
required: true
- label: I checked the documentation and found no answer.
required: true
- label: I have searched the existing issues and made sure this is not a duplicate.
required: true
- label: I have provided sufficient information.
required: true

View file

@ -0,0 +1,77 @@
name: Issue Report
title: "<version> <title>"
description: Use this template to report a problem
labels: [bug]
body:
- type: textarea
id: reproduce-steps
attributes:
label: Steps to reproduce
description: Please provide detailed steps for reproducing the issue.
placeholder: |
Example:
1. Step 1
2. Step 2
3. You get it..
validations:
required: true
- type: textarea
id: failure-logs
attributes:
label: Failure Logs
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
render: shell
validations:
required: true
- type: textarea
id: expected-behavior
attributes:
label: Expected behavior
description: What did you expect to happen?
validations:
required: true
- type: textarea
id: current-behavior
attributes:
label: Current behavior
description: What is the current behavior?
validations:
required: true
- type: dropdown
id: version
attributes:
label: Version
description: What version of the library are you running?
options:
- Default
- Edge
validations:
required: true
- type: textarea
attributes:
label: Anything else?
description: |
Links? References? Anything that will give us more context about the issue you are encountering!
Tip: You can attach images or log files by clicking this area to highlight it and then dragging files in.
validations:
required: false
- type: checkboxes
id: checklist
attributes:
label: Checklist
options:
- label: I am running the latest version.
required: false
- label: I checked the documentation and found no answer.
required: true
- label: I have searched the existing issues and made sure this is not a duplicate.
required: true
- label: I have provided sufficient information.
required: true

View file

@ -0,0 +1,33 @@
name: Question
description: Use this template to ask a question about the project
labels: [question]
body:
- type: textarea
id: question
attributes:
label: Question
placeholder: What do you want to know?
validations:
required: true
- type: textarea
id: other-details
attributes:
label: Other details
placeholder: |
Additional details and attachments.
- type: checkboxes
id: checklist
attributes:
label: Checklist
options:
- label: I am running the latest version.
required: true
- label: I checked the documentation and found no answer.
required: true
- label: I have searched the existing issues and made sure this is not a duplicate.
required: true
- label: I have provided sufficient information.
required: true

View file

@ -0,0 +1,27 @@
# Pull Request Template
## Description
Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
Fixes # (issue)
## Type of change
Please delete options that are not relevant.
- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [ ] This change requires a documentation update
## Checklist:
- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] I have checked my code and corrected any misspellings

17
node_modules/googlethis/.github/workflows/lint.yml generated vendored Normal file
View file

@ -0,0 +1,17 @@
name: Lint
on: [push, pull_request]
jobs:
eslint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
- name: npm install and lint
run: |
npm install
npm run lint

26
node_modules/googlethis/.github/workflows/node.js.yml generated vendored Normal file
View file

@ -0,0 +1,26 @@
name: Build
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [ 14.x, 15.x, 16.x ]
# See supported Node.js release schedule at https://nodejs.org/en/about/releases/
steps:
- uses: actions/checkout@v2
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- run: npm install
- run: npm run test

19
node_modules/googlethis/.github/workflows/stale.yml generated vendored Normal file
View file

@ -0,0 +1,19 @@
name: Mark stale issues and pull requests
on:
schedule:
- cron: "30 1 * * *"
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
stale-issue-message: 'This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.'
stale-pr-message: 'This PR has been automatically marked as stale because it has not had recent activity. Remove the stale label or comment or this will be closed in 2 days'
days-before-stale: 60
days-before-close: 4

21
node_modules/googlethis/LICENSE generated vendored Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 LuanRT
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

2590
node_modules/googlethis/README.md generated vendored Normal file

File diff suppressed because it is too large Load diff

26
node_modules/googlethis/examples/index.js generated vendored Normal file
View file

@ -0,0 +1,26 @@
const google = require('../lib/google_this');
async function start() {
// A simple search
const res = await google.search('Stephen Hawking', {
page: 0,
safe: false,
additional_params: {
hl: 'en'
}
});
console.info('Search Results:', res);
// Image Search
const images = await google.image('The Wolf Among Us', { safe: false });
console.info('Image Search:', images);
// Reverse Image Search
const reverse = await google.search('https://i.pinimg.com/236x/92/16/d9/9216d9a222ef65eb6eabfff1970180d1.jpg', { ris: true });
console.info('Reverse Image Search:', reverse.results);
const news = await google.getTopNews();
console.info('Google Top News:', news);
}
start();

BIN
node_modules/googlethis/images/google_dictionary.png generated vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 65 KiB

BIN
node_modules/googlethis/images/google_translator.png generated vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

BIN
node_modules/googlethis/images/knowledge_panel.png generated vendored Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 279 KiB

7
node_modules/googlethis/jest.config.js generated vendored Normal file
View file

@ -0,0 +1,7 @@
'use strict';
module.exports = {
roots: [ '<rootDir>/test' ],
testMatch: [ '**/*.test.js' ],
testTimeout: 10000
};

313
node_modules/googlethis/lib/core/main.js generated vendored Normal file
View file

@ -0,0 +1,313 @@
'use strict';
const Axios = require('axios');
const Cheerio = require('cheerio');
const Utils = require('../utils/utils');
const Constants = require('../utils/constants');
const OrganicResults = require('./nodes/OrganicResults');
const KnowledgeGraph = require('./nodes/KnowledgeGraph');
const FeaturedSnippet = require('./nodes/FeaturedSnippet');
const Location = require('./nodes/Location');
const Translation = require('./nodes/Translation');
const Dictionary = require('./nodes/Dictionary');
const Converters = require('./nodes/Converters');
const Videos = require('./nodes/Videos');
const TopStories = require('./nodes/TopStories');
const Weather = require('./nodes/Weather');
const Time = require('./nodes/Time');
const PAA = require('./nodes/PAA');
const PAS = require('./nodes/PAS');
const FormData = require('form-data');
/**
* Searches a given query on Google.
* @param {string | object} query - The query to search for.
* @param {object} [options] - Search options.
* @param {boolean} [options.ris] - Weather this is a reverse image search or not.
* @param {boolean} [options.safe] - Weather to use safe search or not.
* @param {number} [options.page] - Page number.
* @param {boolean} [options.parse_ads] - Weather or not to parse ads.
* @param {boolean} [options.use_mobile_ua] - Weather or not to use a mobile user agent.
* @param {object} [options.additional_params] - parameters that will be passed to Google
*/
async function search(query, options = {}) {
let response;
const ris = options.ris || false;
const safe = options.safe || false;
const page = options.page ? options.page * 10 : 0;
const use_mobile_ua = Reflect.has(options, 'use_mobile_ua') ? options.use_mobile_ua : true;
const parse_ads = options.parse_ads || false;
const additional_params = options.additional_params || null;
if (typeof query === 'object' && ris) {
response = await uploadImage(query);
} else {
const _query = query.trim().split(/ +/).join('+');
const url = encodeURI(
ris ?
`${Constants.URLS.W_GOOGLE}searchbyimage?image_url=${_query}`:
`${Constants.URLS.GOOGLE}search?q=${_query}&ie=UTF-8&aomd=1${(safe ? '&safe=active' : '')}&start=${page}`
);
response = await Axios.get(url, {
params: additional_params,
headers: Utils.getHeaders({ mobile: use_mobile_ua })
}).catch((err) => err);
}
if (response instanceof Error)
throw new Utils.SearchError('Could not execute search', {
status_code: response?.status || 0, message: response?.message
});
const $ = Cheerio.load(Utils.refineData(response.data, parse_ads, use_mobile_ua));
const results = {};
results.results = OrganicResults.parse($, parse_ads, use_mobile_ua);
results.videos = Videos.parse($);
results.knowledge_panel = new KnowledgeGraph(response.data, $);
results.featured_snippet = new FeaturedSnippet($);
const did_you_mean = $(Constants.SELECTORS.DID_YOU_MEAN).text();
results.did_you_mean = did_you_mean ? did_you_mean: null;
// These use the same selectors, so we have to check before parsing.
results.weather = new Weather($);
results.time = !results.weather.location ? new Time($): null;
results.location = !results.time?.hours ? new Location($): null;
results.dictionary = new Dictionary($);
results.translation = new Translation($);
results.top_stories = TopStories.parse($);
results.unit_converter = new Converters($);
results.people_also_ask = PAA.parse($, response.data);
results.people_also_search = PAS.parse($);
return results;
}
async function uploadImage(buffer) {
const form_data = new FormData();
form_data.append('encoded_image', buffer);
const response = await Axios.post(`${Constants.URLS.GIS}searchbyimage/upload`, form_data, {
headers: {
...form_data.getHeaders(),
...Utils.getHeaders({ mobile: true })
}
});
return response;
}
/**
* Google image search.
*
* @param {string} query - The query to search for.
* @param {object} [options] - Search options.
* @param {boolean} [options.safe] - Weather to use safe search or not.
* @param {object} [options.additional_params] - Additional parameters that will be passed to Google.
* @returns {Promise.<{
* id: string;
* url: string;
* width: number;
* height: number;
* color: number;
* preview: {
* url: string;
* width: number;
* height: number;
* },
* origin: {
* title: string;
* website: {
* name: string;
* domain: string;
* url: string;
* }
* }
*}[]>}
*/
async function image(query, options = {}) {
const safe = options.safe || false;
const additional_params = options.additional_params || {};
const form_data = new URLSearchParams();
const payload = [
[
[
'HoAMBc',
JSON.stringify([
null, null, [
0, null, 2529, 85, 2396,
[], [9429, 9520], [194, 194],
false, null, null, 9520
],
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null,
null, [
query,
],
null, null, null,
null, null, null,
null, null, [
null, 'CAE=', 'GGwgAA=='
], null, true
]),
null,
'generic'
]
]
];
form_data.append('f.req', JSON.stringify(payload));
form_data.append('at', `${Utils.generateRandomString(29)}:${Date.now()}`);
const params = {
...additional_params
};
if (safe) {
params.safe = 'active';
}
const response = await Axios.post(`${Constants.URLS.W_GOOGLE}_/VisualFrontendUi/data/batchexecute`, form_data, {
params: {
'rpcids': 'HoAMBc',
'source-path': '/search',
'f.sid': -Utils.getRandomInt(0, 9e10),
'bl': 'boq_visualfrontendserver_20220505.05_p0',
'hl': 'en',
'authuser': 0,
'_reqid': -Utils.getRandomInt(0, 9e5),
...params
},
headers: {
'content-type': 'application/x-www-form-urlencoded;charset=UTF-8',
...Utils.getHeaders({ mobile: false })
}
}).catch((err) => err);
if (response instanceof Error)
throw new Utils.SearchError('Could not execute search', {
status_code: response?.response?.status || 0, message: response?.message
});
const res = '[null'+(Utils.getStringBetweenStrings(response.data, '"[null', ']"') || '') + ']';
const data = JSON.parse(res.replace(/\\"/g, '"').replace(/\\\\"/g, '\''));
if (data.length <= 1)
throw new Utils.SearchError('Got unexpected response from BatchExecute API', data);
if (!data[56]?.[1])
throw new Utils.SearchError(data[53]?.[1] || 'Unexpected response structure', data[53]?.[2] || data);
const items = data[56]?.[1]?.[0]?.[0]?.[1]?.[0];
if (!items)
throw new Utils.SearchError('Unexpected response structure', data);
const results = items.map((el) => {
const item = el[0]?.[0]?.['444383007']; // TODO: refactor this
if (!item?.[1])
return;
const image = item[1]?.[3];
const preview = item[1]?.[2];
const origin = item[1]?.[9];
if (image && preview && origin)
return {
id: item[1][1],
url: decodeURIComponent(JSON.parse('"' + image[0].replace(/"/g, '"') + '"')),
width: image[1],
height: image[2],
color: item[1][6],
preview: {
url: decodeURIComponent(JSON.parse('"' + preview[0].replace(/"/g, '"') + '"')),
width: preview[1],
height: preview[2]
},
origin: {
title: origin['2008'][1],
website: {
name: origin['2003'][12],
domain: origin['2003'][17],
url: origin['2003'][2]
}
}
}
}).filter((item) => item);
return results;
}
/**
* Retrieves news from Google.
*
* @param {string} [language] - two digits language code.
* @param {string} [region] - two digits region code.
*
* @returns {Promise.<{
* headline_stories: {
* title: string;
* url: string;
* image: string;
* published: string;
* by: string;
* }[]
* }>}
*/
async function getTopNews(language = 'en', region = 'US') {
const url = Constants.URLS.GOOGLE_NEWS + `topstories?tab=in&hl=${language.toLocaleLowerCase()}-${region.toLocaleUpperCase()}&gl=${region.toLocaleUpperCase()}&ceid=${region.toLocaleUpperCase()}:${language.toLocaleLowerCase()}`;
const response = await Axios.get(url,
{
headers: Utils.getHeaders({
mobile: true
})
}).catch((err) => err);
if (response instanceof Error) throw new Error('Could not retrieve top news: ' + response.message);
const $ = Cheerio.load(response.data);
const results = {
headline_stories: []
};
const headline_stories_publishers = $(Constants.SELECTORS.PUBLISHER).map((i, el) => $(el).text()).get();
const headline_stories_imgs = $(Constants.SELECTORS.STORY_IMG).map((i, el) => $(el).attr('src')).get();
const headline_stories_time = $(Constants.SELECTORS.STORY_TIME).map((i, el) => $(el).text()).get();
$(Constants.SELECTORS.STORY_TITLE).each((i, el) => {
const headline_stories_title = $(el).text();
const headline_stories_url = $(el).attr('href');
results.headline_stories.push({
title: headline_stories_title,
url: `${Constants.URLS.GOOGLE_NEWS}${headline_stories_url.slice(2)}`,
image: headline_stories_imgs[i],
published: headline_stories_time[i],
by: headline_stories_publishers[i]
});
});
return results;
}
module.exports = {
getTopNews,
search,
image
};

34
node_modules/googlethis/lib/core/nodes/Converters.js generated vendored Normal file
View file

@ -0,0 +1,34 @@
'use strict';
const Constants = require('../../utils/constants');
class Converters {
constructor($) {
const unit_converter_input = $(Constants.SELECTORS.UNIT_CONVERTER_INPUT).attr('value');
const unit_converter_output = $(Constants.SELECTORS.UNIT_CONVERTER_OUTPUT).attr('value');
const unit_converter_formula = $(Constants.SELECTORS.UNIT_CONVERTER_FORMULA).text();
const input_currency_name = $(Constants.SELECTORS.INPUT_CURRENCY_NAME).attr('data-name');
const output_currency_name = $(Constants.SELECTORS.OUTPUT_CURRENCY_NAME).attr('data-name');
const currency_converter_input = $(Constants.SELECTORS.CURRENCY_CONVERTER_INPUT).text();
const currency_converter_output = $(Constants.SELECTORS.CURRENCY_CONVERTER_OUTPUT).text();
if (unit_converter_input && unit_converter_output) {
this.input = unit_converter_input;
this.output = unit_converter_output;
this.formula = unit_converter_formula;
} else if (currency_converter_input && currency_converter_output) {
this.input = {
name: input_currency_name,
value: currency_converter_input
}
this.output = {
name: output_currency_name,
value: currency_converter_output
}
}
}
}
module.exports = Converters;

34
node_modules/googlethis/lib/core/nodes/Dictionary.js generated vendored Normal file
View file

@ -0,0 +1,34 @@
'use strict';
const Constants = require('../../utils/constants');
class Dictionary {
/** @type {string | null} */
word;
/** @type {string | null} */
phonetic;
/** @type {string | null} */
audio;
/** @type {string[]} */
definitions;
/** @type {string[]} */
examples;
constructor($) {
const word = $(Constants.SELECTORS.GD_WORD).text();
const phonetic = $(Constants.SELECTORS.GD_PHONETIC).text();
const audio = $(Constants.SELECTORS.GD_AUDIO).attr('src');
this.word = word || null;
this.phonetic = word ? phonetic || 'N/A' : null;
this.audio = word && audio ? `https:${audio}` : null;
this.definitions = word ? $(Constants.SELECTORS.GD_DEFINITIONS).map((i, el) => $(el).text()).get() : [];
this.examples = word ? $(Constants.SELECTORS.GD_EXAMPLES).map((i, el) => $(el).text()).get() : [];
}
}
module.exports = Dictionary;

View file

@ -0,0 +1,41 @@
'use strict';
const Constants = require('../../utils/constants');
class FeaturedSnippet {
/** @type {string | null} */
title;
/** @type {string | null} */
description;
/** @type {string | null} */
url;
constructor($) {
const featured_snippet_title =
$(Constants.SELECTORS.FEATURED_SNIPPET_TITLE[0]).text() ||
$(Constants.SELECTORS.FEATURED_SNIPPET_TITLE[1]).text() ||
$(Constants.SELECTORS.FEATURED_SNIPPET_TITLE[2]).text();
const featured_snippet_url = $(Constants.SELECTORS.FEATURED_SNIPPET_URL).map((i, el) => $(el).attr('href')).get()[0];
const featured_snippet = Constants.SELECTORS.FEATURED_SNIPPET_DESC.map((selector) => {
if ($(selector)[0] && selector != Constants.SELECTORS.FEATURED_SNIPPET_DESC[2]) {
return $(selector).html()
.replace(/<\/li>|<\/b>|<b>/g, '')
.replace(/&amp;/g, '&')
.split('<li class="TrT0Xe">')
.join('\n').trim();
} else if (selector == Constants.SELECTORS.FEATURED_SNIPPET_DESC[2]) {
return $(selector).text();
}
}).filter(text => text && text.length)[0];
this.title = featured_snippet_title || null;
this.description = featured_snippet || null;
this.url = featured_snippet_url || null;
}
}
module.exports = FeaturedSnippet;

View file

@ -0,0 +1,171 @@
'use strict';
const Utils = require('../../utils/utils');
const Constants = require('../../utils/constants');
class MetadataItem {
/** @type {string} */
title;
/** @type {string} */
value;
constructor (data) {
this.title = data.title;
this.value = data.value;
}
}
class Social {
/** @type {string} */
name;
/** @type {string} */
url;
/** @type {string} */
icon;
constructor (data) {
this.name = data.name;
this.url = data.url;
this.icon = data.icon;
}
}
class KnowledgeGraph {
/** @type {string | null} */
type;
/** @type {string | null} */
title;
/** @type {string | null} */
description;
/** @type {string | null} */
url;
/** @type {{ title: string; value: string }[]} */
metadata = [];
/** @type {{ title: string; year: string; }[]} */
books = [];
/** @type {{ title: string; year: string; }[]} */
tv_shows_and_movies = [];
ratings = [];
/** @type {string[]} */
available_on = [];
/** @type {{ url: string; source: string }[]} */
images = [];
/** @type {{ title: string; album: string }[]} */
songs = [];
/** @type {Social[]} */
socials;
/** @type {string | null} */
demonstration;
/** @type {string | null} */
lyrics;
constructor (data, $) {
this.title = $(Constants.SELECTORS.KNO_PANEL_TITLE[0]).first().text() || $(Constants.SELECTORS.KNO_PANEL_TITLE[1]).text() || null;
this.description = $(Constants.SELECTORS.KNO_PANEL_DESCRIPTION).first().text() || null;
this.url = $(Constants.SELECTORS.KNO_PANEL_URL).attr('href') || null;
// Extract metadata from the knowledge graph
$(Constants.SELECTORS.KNO_PANEL_METADATA).each((_i, el) => {
const key = $(el).first().text().trim().slice(0, -1);
const value = $(el).next().text().trim();
if (value.length) {
this.metadata.push(new MetadataItem({ title: key, value: value.trim() }));
}
});
const knowledge_panel_type = $(Constants.SELECTORS.KNO_PANEL_TYPE).last().text();
if (knowledge_panel_type && knowledge_panel_type !== this.title) {
this.type = knowledge_panel_type;
} else {
this.type = null;
}
this.books = $(Constants.SELECTORS.KNO_PANEL_BOOKS).map((_i, el) => {
const title = $(el).first().text().trim();
const year = $(el).next().text().trim();
if (year.length)
return { title, year }
}).get();
this.tv_shows_and_movies = $(Constants.SELECTORS.KNO_PANEL_TV_SHOWS_AND_MOVIES).map((_i, el) => {
const title = $(el).first().text().trim();
const year = $(el).next().text().trim();
if (year.length)
return { title, year };
}).get();
const lyrics = $(Constants.SELECTORS.KNO_PANEL_SONG_LYRICS)
.map((i, el) =>
$($(el).html()
.replace(/<br aria-hidden="true">/g, '\n')
.replace(/<\/span><\/div><div jsname=".*" class=".*"><span jsname=".*">/g, '\n\n')
.replace(/<br>/g, '\n')).text()).get();
this.lyrics = lyrics.length ? lyrics.join('\n\n') : null;
const google_users_rating = $(Constants.SELECTORS.KNO_PANEL_FILM_GOOGLEUSERS_RATING)[0];
if (google_users_rating) {
const rating = $(google_users_rating.children[0].children[0]).text() || null;
this.ratings.push({
name: 'Google Users',
rating: rating
});
}
$(Constants.SELECTORS.KNO_PANEL_FILM_RATINGS[0]).each((i, el) => {
const name = $($(Constants.SELECTORS.KNO_PANEL_FILM_RATINGS[1])[i]).attr('title');
const rating = $(el).text();
this.ratings.push({ name, rating });
});
this.available_on = $(Constants.SELECTORS.KNO_PANEL_AVAILABLE_ON).map((_i, el) => $(el).text()).get();
this.images = $(Constants.SELECTORS.KNO_PANEL_IMAGES).map((_i, elem) => {
const url = $(elem).attr('data-src');
const source = $(elem).parent().parent().parent().parent().parent().attr('data-lpage');
return { url, source };
}).get().filter((img) => img.url);
this.songs = $(Constants.SELECTORS.KNO_PANEL_SONGS).map((_i, el) => {
const title = $(el).text().trim();
const album = $(el).next().find('span').first().text().trim();
return { title, album };
}).get();
this.socials = $(Constants.SELECTORS.KNO_PANEL_SOCIALS).map((_i, el) => {
const name = $(el).text();
const url = $(el).attr('href');
const icon = $(el).find('img').attr('src');
return new Social({ name, url, icon });
}).get();
const demo = Utils.getStringBetweenStrings(data, 'source src\\x3d\\x22', '.mp4');
this.demonstration = demo ? `${demo}.mp4` : null;
}
}
module.exports = KnowledgeGraph;

28
node_modules/googlethis/lib/core/nodes/Location.js generated vendored Normal file
View file

@ -0,0 +1,28 @@
'use strict';
const Constants = require('../../utils/constants');
class Location {
/** @type {string | null} */
title;
/** @type {string | null} */
distance;
/** @type {string | null} */
map;
constructor($) {
const location_title = $(Constants.SELECTORS.LOCATION_TITLE).text();
const location_distance = $(Constants.SELECTORS.LOCATION_DISTANCE).text();
const location_image = $(Constants.SELECTORS.LOCATION_IMAGE).attr('src');
const is_available = location_title && location_distance;
this.title = is_available ? location_title : null;
this.distance = is_available ? location_distance : null;
this.map = is_available ? `https://google.com/${location_image}` : null;
}
}
module.exports = Location;

View file

@ -0,0 +1,146 @@
'use strict';
const Constants = require('../../utils/constants');
class OrganicResult {
/** @type {string} */
title;
/** @type {string} */
description;
/** @type {string} */
url;
/** @type {boolean} */
is_sponsored;
/** @type {{ high_res: string; low_res: string; }} */
favicons;
constructor(data) {
this.title = data.title;
this.description = data.description;
this.url = data.url;
this.is_sponsored = data.is_sponsored;
this.favicons = data.favicons;
}
}
class OrganicResults {
/**
* @returns {{
* title: string;
* description: string;
* url: string;
* is_sponsored: boolean;
* favicons: {
* high_res: string;
* low_res: string;
* }
* }[]}
*/
static parse($, parse_ads = false, is_mobile = true) {
// Stores advert indexes so we can identify them later
const ad_indexes = [];
const titles = $(Constants.SELECTORS.TITLE)
.map((_i, el) => {
const is_ad =
el.parent.attribs.style == '-webkit-line-clamp:2' ||
(!is_mobile && el.parent.attribs.class.startsWith('vdQmEd'));
// Ignore ad titles if parse_ads is false
if (!parse_ads && is_ad)
return null;
return is_mobile ?
$(el).text().trim() : $(el).find('h3').text().trim() || $(el).find('a > div > span').first().text().trim();
}).get();
const descriptions = $(Constants.SELECTORS.DESCRIPTION)
.map((_i, el) => {
const is_ad = el.parent.attribs.class == 'w1C3Le' ||
(!is_mobile && !Object.keys(el.parent.attribs).length);
// Ignore ad descriptions if parse_ads is false
if (!parse_ads && is_ad) {
return null;
} else if (is_ad) {
ad_indexes.push(_i);
}
return $(el).text().trim();
}).get();
const urls = $(is_mobile ? Constants.SELECTORS.URL : `${Constants.SELECTORS.TITLE} > a`)
.map((_i, el) => {
const is_ad = el.parent?.parent?.attribs?.class?.startsWith('vdQmEd');
/**
* Since the selector for URLs is the same as the one for titles on desktop,
* we need to check if the element is an ad. If we're parsing the mobile page,
* then ads are simply stripped out of the results.
*/
if (!is_mobile && !parse_ads && is_ad) {
return null;
}
return $(el).attr('href');
}).get();
// Refine results
if (titles.length < urls.length && titles.length < descriptions.length) {
urls.shift();
}
if (urls.length > titles.length) {
urls.shift();
}
const is_innacurate_data = descriptions.length < urls.slice(1).length;
urls.forEach((item, index) => {
// Why YouTube? Because video results usually don't have a description.
if (item.includes('m.youtube.com') && is_innacurate_data) {
urls.splice(index, 1);
titles.splice(index, 1);
index--;
}
});
const results = [];
for (let i = 0; i < titles.length; i++) {
const title = titles[i];
const description = descriptions[i];
let url = urls[i];
// Some results have a different URL format (AMP and ad results).
if (url?.startsWith('/aclk') || url?.startsWith('/amp/s')) {
url = `${Constants.URLS.W_GOOGLE}${url.substring(1)}`;
}
const high_res_favicon = `${Constants.URLS.FAVICONKIT}/${new URL(url || Constants.URLS.W_GOOGLE).hostname}/192`;
const low_res_favicon = `${Constants.URLS.W_GOOGLE}s2/favicons?sz=64&domain_url=${new URL(url || Constants.URLS.W_GOOGLE).hostname}`;
if (titles[i] && descriptions[i] && urls[i]) {
results.push(new OrganicResult({
title,
description,
url,
is_sponsored: ad_indexes.includes(i),
favicons: {
high_res: high_res_favicon,
low_res: low_res_favicon
}
}));
}
}
return results;
}
}
module.exports = OrganicResults;

28
node_modules/googlethis/lib/core/nodes/PAA.js generated vendored Normal file
View file

@ -0,0 +1,28 @@
'use strict';
const unraw = require('unraw').default;
const Utils = require('../../utils/utils');
const Constants = require('../../utils/constants');
class PAA {
static parse($, data) {
/** @type {string[]} */
const items = [];
Constants.SELECTORS.PAA.forEach((item) =>
$(item).each((i, el) => items.push($(el).text())));
items.shift();
const extra_data = JSON.parse(unraw(Utils.getStringBetweenStrings(data, 'var c=\'', '\';google') || '{}'));
const rfs = extra_data?.sb_wiz?.rfs;
if (rfs) {
rfs.forEach((el) => items.push(el.replace(/<b>|<\/b>/g, '')));
}
return items;
}
}
module.exports = PAA;

33
node_modules/googlethis/lib/core/nodes/PAS.js generated vendored Normal file
View file

@ -0,0 +1,33 @@
'use strict';
const Constants = require('../../utils/constants');
class Query {
/** @type {string} */
title;
/** @type {string} */
thumbnail;
constructor (data) {
this.title = data.title;
this.thumbnail = data.thumbnail;
}
}
class PAS {
static parse($) {
/** @type {{ title: string; thumbnail: string }[]} */
const items = [];
$(Constants.SELECTORS.PASF).each((i, el) => {
if ($(el).attr('data-src')) {
items.push(new Query({ title: $(el).attr('alt'), thumbnail: `https:${$(el).attr('data-src')}` }));
}
});
return items;
}
}
module.exports = PAS;

21
node_modules/googlethis/lib/core/nodes/Time.js generated vendored Normal file
View file

@ -0,0 +1,21 @@
'use strict';
const Constants = require('../../utils/constants');
class Time {
/** @type {string | null} */
hours;
/** @type {string | null} */
date;
constructor($) {
const hours = $(Constants.SELECTORS.CURRENT_TIME_HOUR).first().text();
const date = $(Constants.SELECTORS.CURRENT_TIME_DATE).map((i, el) => $(el).text()).get()[1];
this.hours = date ? hours.trim() : null;
this.date = date ? date.trim() : null;
}
}
module.exports = Time;

46
node_modules/googlethis/lib/core/nodes/TopStories.js generated vendored Normal file
View file

@ -0,0 +1,46 @@
'use strict';
const Constants = require('../../utils/constants');
class Item {
/** @type {string} */
description;
/** @type {url} */
url;
constructor(data) {
this.description = data.description;
this.url = data.url;
}
}
class TopStories {
static parse($) {
// Removes unwanted text from the description
$(`${Constants.SELECTORS.TOP_STORIES_DESCRIPTION[0]} > div.CEMjEf`).each((el) => $(el).remove());
$(`${Constants.SELECTORS.TOP_STORIES_DESCRIPTION[0]} > div > p`).each((el) => $(el).remove());
const top_stories_descriptions = Constants.SELECTORS.TOP_STORIES_DESCRIPTION.map((selector) =>
$(selector).map((el) => $(el).text()).get()).filter((descs) => descs.length > 0)[0];
const top_stories_urls = $(Constants.SELECTORS.TOP_STORIES_URL).map((el) => $(el).attr('href')).get();
/** @type {{
description: string;
url: string;
}[]} */
const items = [];
if (top_stories_descriptions) {
for (let i = 0; i < top_stories_urls.length; i++) {
items.push(new Item({
description: top_stories_descriptions[i], url: top_stories_urls[i]
}));
}
}
return items;
}
}
module.exports = TopStories;

32
node_modules/googlethis/lib/core/nodes/Translation.js generated vendored Normal file
View file

@ -0,0 +1,32 @@
'use strict';
const Constants = require('../../utils/constants');
class Translation {
/** @type {string | null} */
source_language;
/** @type {string | null} */
target_language;
/** @type {string | null} */
source_text;
/** @type {string | null} */
target_text;
constructor($) {
const source_language = $(Constants.SELECTORS.TR_SOURCE_LANGUAGE).text();
const target_language = $(Constants.SELECTORS.TR_TARGET_LANGUAGE).text();
const source_text = $(Constants.SELECTORS.TR_SOURCE_TEXT).text();
const target_text = $(Constants.SELECTORS.TR_TARGET_TEXT).text();
this.source_language = source_text.length ? source_language : null;
this.target_language = source_text.length ? target_language : null;
this.source_text = source_text.length ? source_text : null;
this.target_text = target_text.length ? target_text : null;
}
}
module.exports = Translation;

61
node_modules/googlethis/lib/core/nodes/Videos.js generated vendored Normal file
View file

@ -0,0 +1,61 @@
'use strict';
const Constants = require('../../utils/constants');
class Video {
/** @type {string} */
id;
/** @type {string} */
url;
/** @type {string} */
title;
/** @type {string} */
author;
/** @type {string} */
duration;
constructor(data) {
this.id = data.id;
this.url = data.url;
this.title = data.title;
this.author = data.author;
this.duration = data.duration;
}
}
class Videos {
static parse($) {
const data = $(Constants.SELECTORS.VIDEOS);
/**
* @type {{
* id: string;
* url: string;
* title: string;
* author: string;
* duration: string;
* }[]}
*/
const videos = [];
for (const elem of data) {
const id = $(elem).find('a').attr('href')?.split('v=')?.[1];
const url = $(elem).find('a').attr('href');
const title = $(elem).children().find('a > div > div').prev().text().trim();
const author = $(elem).children().find('a > div > div > span').next().next().prev().text().replace(/·/g, '').trim();
const duration = $(elem).children().find('div[role="presentation"]').first().text();
if (id && url && title && author && duration) {
videos.push(new Video({ id, url, title, author, duration }));
}
}
return videos;
}
}
module.exports = Videos;

43
node_modules/googlethis/lib/core/nodes/Weather.js generated vendored Normal file
View file

@ -0,0 +1,43 @@
'use strict';
const Constants = require('../../utils/constants');
class Weather {
/** @type {string | null} */
location;
/** @type {string | null} */
forecast;
/** @type {string | null} */
precipitation;
/** @type {string | null} */
humidity;
/** @type {string | null} */
temperature;
/** @type {string | null} */
wind;
constructor($) {
const weather_location = $(Constants.SELECTORS.WEATHER_LOCATION).text();
const weather_forecast = $(Constants.SELECTORS.WEATHER_FORECAST).text();
const precipitation = $(Constants.SELECTORS.PRECIPITATION).text();
const air_humidity = $(Constants.SELECTORS.AIR_HUMIDITY).text();
const temperature = $(Constants.SELECTORS.TEMPERATURE).text();
const wind_speed = $(Constants.SELECTORS.WIND_SPEED).text();
const is_available = weather_location && weather_forecast;
this.location = is_available ? weather_location : null;
this.forecast = is_available ? weather_forecast : null;
this.precipitation = is_available ? precipitation : null;
this.humidity = is_available ? air_humidity : null;
this.temperature = is_available ? temperature : null;
this.wind = is_available ? wind_speed : null;
}
}
module.exports = Weather;

1
node_modules/googlethis/lib/index.js generated vendored Normal file
View file

@ -0,0 +1 @@
module.exports = require('./core/main');

101
node_modules/googlethis/lib/utils/constants.js generated vendored Normal file
View file

@ -0,0 +1,101 @@
'use strict';
module.exports = {
URLS: {
GIS: 'https://images.google.com/',
GOOGLE: 'https://google.com/',
W_GOOGLE: 'https://www.google.com/',
GOOGLE_NEWS: 'https://news.google.com/',
FAVICONKIT: 'https://api.faviconkit.com'
},
SELECTORS: {
// Organic Search Results
TITLE: 'div.ynAwRc.q8U8x.MBeuO.gsrt.oewGkc.LeUQr',
DESCRIPTION: 'div.MUxGbd.yDYNvb',
URL: 'a.C8nzq.BmP5tf',
// Did You Mean
DID_YOU_MEAN: 'a.gL9Hy',
// Knowledge Panel
KNO_PANEL_TITLE: [ 'div.BkwXh > div', 'div > span.u9DLmf' ],
KNO_PANEL_DESCRIPTION: 'div[class="kno-rdesc"] > span',
KNO_PANEL_URL: 'div[class="kno-rdesc"] > span > a',
KNO_PANEL_METADATA: 'div.rVusze > span',
KNO_PANEL_TYPE: 'div.BkwXh > div',
KNO_PANEL_SONG_LYRICS: 'div.ujudUb',
KNO_PANEL_AVAILABLE_ON: 'div[class="ellip bclEt"]',
KNO_PANEL_IMAGES: 'g-inner-card > div > div > img',
KNO_PANEL_SONGS: 'a > div > div > div > div[class="title"]',
KNO_PANEL_BOOKS: 'div[data-attrid="kc:/book/author:books only"] > a > div > div > div.Bo9xMe > div',
KNO_PANEL_TV_SHOWS_AND_MOVIES: 'div[data-attrid="kc:/people/person:tv-shows-and-movies"] > a > div > div > div.Bo9xMe > div',
KNO_PANEL_FILM_GOOGLEUSERS_RATING: 'div[data-attrid="kc:/ugc:thumbs_up"] > div > div > div',
KNO_PANEL_FILM_RATINGS: ['span[class="gsrt KMdzJ"]', 'span[class="rhsB pVA7K"]'],
KNO_PANEL_SOCIALS: 'div[data-attrid="kc:/common/topic:social media presence"] > div > kp-carousel > g-scrolling-carousel > div > div > kp-carousel-item > div > g-link > a',
VIDEOS: 'div > div > div > div > video-voyager > div',
// Featured Snippet
FEATURED_SNIPPET_TITLE: ['div[class="co8aDb gsrt"]', 'a[class="sXtWJb gsrt"]', 'div[class="Xv4xee"]'],
FEATURED_SNIPPET_DESC: ['ol[class="X5LH0c"]', 'ul[class="i8Z77e"]', 'div[data-attrid="wa:/description"]'],
FEATURED_SNIPPET_URL: 'div > div > h3 > a',
// Unit converter
UNIT_CONVERTER_INPUT: 'div.rpnBye > input',
UNIT_CONVERTER_OUTPUT: 'div[id="NotFQb"] > input',
UNIT_CONVERTER_FORMULA: 'div.bjhkR',
INPUT_CURRENCY_NAME: 'span.vLqKYe',
OUTPUT_CURRENCY_NAME: 'span.MWvIVe',
CURRENCY_CONVERTER_INPUT: 'span.DFlfde.eNFL1',
CURRENCY_CONVERTER_OUTPUT: 'span.DFlfde.SwHCTb',
// Weather forecast
WEATHER_LOCATION: 'div.wob_hdr > div[id="wob_loc"]',
WEATHER_FORECAST: 'div.wob_dsc',
PRECIPITATION: 'div.wob_dtf > div > span[id="wob_pp"]',
AIR_HUMIDITY: 'div.wob_dtf > div > span[id="wob_hm"]',
TEMPERATURE: 'div > span[id="wob_tm"]',
WIND_SPEED: 'span[id="wob_ws"]',
// Time result, E.g: try searching “what time is it in Japan?”
CURRENT_TIME_HOUR: 'div > div[role="heading"]',
CURRENT_TIME_DATE: 'div.vk_gy.vk_sh',
// Location result
LOCATION_TITLE: 'div.vk_sh.vk_gy',
LOCATION_DISTANCE: 'div.dDoNo.FzvWSb.vk_bk',
LOCATION_IMAGE: 'div.vk_c > div > a > img',
// Google Dictionary
GD_WORD: 'span[data-dobid="hdw"]',
GD_PHONETIC: 'div.qexShd',
GD_AUDIO: 'audio > source',
GD_DEFINITIONS: 'div[data-dobid="dfn"]',
GD_EXAMPLES: 'div[class="ubHt5c"]',
// Google Translator
TR_SOURCE_LANGUAGE: 'div[class="j1iyq"] > span[class="source-language"]',
TR_TARGET_LANGUAGE: 'div[class="j1iyq"] > span[class="target-language"]',
TR_SOURCE_TEXT: 'pre[id="tw-source-text"] > span[class="Y2IQFc"]',
TR_TARGET_TEXT: 'pre[id="tw-target-text"] > span[class="Y2IQFc"]',
// Top Stories
TOP_STORIES_DESCRIPTION: ['div.g5wfEd', 'div.VeOk3'],
TOP_STORIES_URL: 'a.WlydOe.amp_r',
TOP_STORIES_SNIPPET: 'div[class="g5wfEd"] > div[role="heading"]',
TOP_STORIES_WEBSITE: 'div[class="g5wfEd"] > div > g-img > img',
// “People also ask”
PAA: [ 'div.s75CSd.u60jwe.gduDCb > span', 'div.gbCQS.u60jwe.gduDCb > div > span', 'div.JlqpRe > span' ],
// “People also search for”
PASF: 'div[class="IHdOHf"] > img',
// Top News
PUBLISHER: 'a[data-n-tid="9"]',
STORY_TITLE: 'a[class="DY5T1d RZIKme"]',
STORY_IMG: 'img[class="tvs3Id QwxBBf"]',
STORY_TIME: 'time'
}
};

42
node_modules/googlethis/lib/utils/user-agents.json generated vendored Normal file
View file

@ -0,0 +1,42 @@
{
"desktop": [
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36 Edg/103.0.1264.62",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36 Edg/103.0.1264.49",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"
],
"mobile": [
"Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 11; SM-G781B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 12; SM-G973F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 11; Redmi Note 8 Pro) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 11; ONEPLUS A6013) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
"Mozilla/5.0 (Linux; Android 12; SM-G986B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/17.0 Chrome/96.0.4664.104 Mobile Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/103.0.5060.63 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/103.0.5060.63 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (iPhone; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.25 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
]
}

131
node_modules/googlethis/lib/utils/utils.js generated vendored Normal file
View file

@ -0,0 +1,131 @@
'use strict';
const userAgents = require('./user-agents.json');
class SearchError extends Error {
constructor (message, info) {
super(message);
info && (this.info = info);
this.date = new Date();
this.version = require('../../package.json').version;
}
}
/**
* Returns headers with a random user agent.
*
* @param {boolean} is_mobile
* @returns {string}
*/
function getHeaders(options = { mobile: false }) {
const available_agents = userAgents[options.mobile ? 'mobile' : 'desktop'];
const ua = available_agents[Math.floor(Math.random() * available_agents.length)];
return {
'accept': 'text/html',
'accept-encoding': 'gzip, deflate',
'accept-language': 'en-US,en',
'referer': 'https://www.google.com/',
'upgrade-insecure-requests': 1,
'user-agent': ua
};
}
/**
* Refines the html.
*
* @param {string} data - Raw html data.
* @param {boolean} parse_ads - Whether to parse ads or not.
* @returns {string}
*/
function refineData (data, parse_ads = false, is_mobile = true) {
let result = data
// Removes classes we don't need:
.replace(/N6jJud MUxGbd lyLwlc/g, '')
.replace(/YjtGef ExmHv MUxGbd/g, '')
.replace(/MUxGbd lyLwlc aLF0Z/g, '')
/*
* Transforms all possible variations of some classes' name into a
* fixed string so it's easier to get consistent results:
**/
// Descriptions: -> MUxGbd yDYNvb
.replace(/yDYNvb lEBKkf/g, 'yDYNvb')
.replace(/VwiC3b MUxGbd yDYNvb/g, 'MUxGbd yDYNvb')
// Urls: -> C8nzq BmP5tf
.replace(/cz3goc BmP5tf/g, 'C8nzq BmP5tf')
// Titles: -> ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr
.replace(/ynAwRc q8U8x MBeuO oewGkc LeUQr/g, 'ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr')
.replace(/MBeuO oewGkc/g, 'MBeuO gsrt oewGkc');
// Transform desktop title/urls classes. Everything else is the same.
if (!is_mobile) {
result = result
.replace(/yuRUbf|v5yQqb/g, 'ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr')
}
// Transform ad title classes.
if (parse_ads) {
result = result
.replace(/cz3goc v5yQqb BmP5tf/g, 'C8nzq BmP5tf')
}
return result;
}
/**
* Gets a string between two delimiters.
*
* @param {string} data - The data.
* @param {string} start_string - Start string.
* @param {string} end_string - End string.
*
* @returns {string}
*/
function getStringBetweenStrings (data, start_string, end_string) {
const regex = new RegExp(`${escapeStringRegexp(start_string)}(.*?)${escapeStringRegexp(end_string)}`, 's');
const match = data.match(regex);
return match ? match[1] : undefined;
}
function escapeStringRegexp (string) {
return string.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&').replace(/-/g, '\\x2d');
}
/**
* Generates a random string with a given length.
* @param {number} length
* @returns {string}
*/
function generateRandomString(length) {
const result = [];
const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_';
for (let i = 0; i < length; i++) {
result.push(alphabet.charAt(Math.floor(Math.random() * alphabet.length)));
}
return result.join('');
}
/**
* Returns a random integer between two values.
*
* @param {number} min
* @param {number} max
*
* @returns {number}
*/
function getRandomInt(min, max) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min) + min); //The maximum is exclusive and the minimum is inclusive
}
module.exports = { SearchError, getHeaders, getStringBetweenStrings, generateRandomString, getRandomInt, refineData };

61
node_modules/googlethis/package.json generated vendored Normal file
View file

@ -0,0 +1,61 @@
{
"name": "googlethis",
"version": "1.6.0",
"description": "A simple yet powerful module to retrieve organic search results and much more from Google.",
"main": "lib/index.js",
"author": "LuanRT <luan.lrt4@gmail.com> (https://github.com/LuanRT)",
"funding": "https://ko-fi.com/luanrt",
"license": "MIT",
"scripts": {
"test": "jest",
"lint": "eslint ./",
"lint:fix": "eslint --fix ./",
"build:types": "npx tsc"
},
"engines": {
"node": ">=14"
},
"types": "./typings/index.d.ts",
"directories": {
"test": "./test",
"typings": "./typings",
"examples": "./examples",
"lib": "./lib"
},
"dependencies": {
"axios": "^0.21.1",
"cheerio": "1.0.0-rc.10",
"form-data": "^4.0.0",
"unraw": "^2.0.1"
},
"repository": {
"type": "git",
"url": "git+https://github.com/LuanRT/google-this.git"
},
"keywords": [
"search",
"google",
"google-this",
"google-search",
"google-scraper",
"google-image-search",
"reverse-image-search",
"search-results",
"web-scraping",
"image-search",
"image",
"scraper",
"serp",
"gis"
],
"bugs": {
"url": "https://github.com/LuanRT/google-this/issues"
},
"homepage": "https://github.com/LuanRT/google-this#readme",
"devDependencies": {
"@types/node": "^17.0.31",
"eslint": "^8.16.0",
"jest": "^28.1.0",
"typescript": "^4.6.4"
}
}

30
node_modules/googlethis/test/main.test.js generated vendored Normal file
View file

@ -0,0 +1,30 @@
'use strict';
const google = require('..');
describe('GoogleThis Tests', () => {
it('Should search a query', async () => {
const search = await google.search('Stephen Hawking');
expect(search.results).not.toHaveLength(0);
});
it('Should search using a desktop user agent', async () => {
const search = await google.search('Stephen Hawking', { use_mobile_ua: false });
expect(search.results).not.toHaveLength(0);
});
it('Should search images', async () => {
const search = await google.image('Supermassive Blackhole');
expect(search).not.toHaveLength(0);
});
it('Should do reverse image search', async () => {
const search = await google.search('https://i.pinimg.com/236x/92/16/d9/9216d9a222ef65eb6eabfff1970180d1.jpg', { ris: true });
expect(search.results).not.toHaveLength(0);
});
it('Should retrieve top news', async () => {
const news = await google.getTopNews('en', 'AU');
expect(news.headline_stories).not.toHaveLength(0);
});
});

19
node_modules/googlethis/tsconfig.json generated vendored Normal file
View file

@ -0,0 +1,19 @@
{
"compilerOptions": {
"declaration": true,
"emitDeclarationOnly": true,
"allowJs": true,
"outDir": "./typings",
"lib": ["ESNext"],
"target": "ESNext",
"moduleResolution": "node"
},
"include": [
"./lib/**/*.js",
"./index.js"
],
"exclude": [
"node_modules",
"**/*.d.ts"
]
}

139
node_modules/googlethis/typings/core/main.d.ts generated vendored Normal file
View file

@ -0,0 +1,139 @@
/**
* Retrieves news from Google.
*
* @param {string} [language] - two digits language code.
* @param {string} [region] - two digits region code.
*
* @returns {Promise.<{
* headline_stories: {
* title: string;
* url: string;
* image: string;
* published: string;
* by: string;
* }[]
* }>}
*/
export function getTopNews(language?: string, region?: string): Promise<{
headline_stories: {
title: string;
url: string;
image: string;
published: string;
by: string;
}[];
}>;
/**
* Searches a given query on Google.
* @param {string | object} query - The query to search for.
* @param {object} [options] - Search options.
* @param {boolean} [options.ris] - Weather this is a reverse image search or not.
* @param {boolean} [options.safe] - Weather to use safe search or not.
* @param {number} [options.page] - Page number.
* @param {boolean} [options.parse_ads] - Weather or not to parse ads.
* @param {boolean} [options.use_mobile_ua] - Weather or not to use a mobile user agent.
* @param {object} [options.additional_params] - parameters that will be passed to Google
*/
export function search(query: string | object, options?: {
ris?: boolean;
safe?: boolean;
page?: number;
parse_ads?: boolean;
use_mobile_ua?: boolean;
additional_params?: object;
}): Promise<{
results: {
title: string;
description: string;
url: string;
is_sponsored: boolean;
favicons: {
high_res: string;
low_res: string;
};
}[];
videos: {
id: string;
url: string;
title: string;
author: string;
duration: string;
}[];
knowledge_panel: KnowledgeGraph;
featured_snippet: FeaturedSnippet;
did_you_mean: string;
weather: Weather;
time: Time;
location: Location;
dictionary: Dictionary;
translation: Translation;
top_stories: {
description: string;
url: string;
}[];
unit_converter: Converters;
people_also_ask: string[];
people_also_search: {
title: string;
thumbnail: string;
}[];
}>;
/**
* Google image search.
*
* @param {string} query - The query to search for.
* @param {object} [options] - Search options.
* @param {boolean} [options.safe] - Weather to use safe search or not.
* @param {object} [options.additional_params] - Additional parameters that will be passed to Google.
* @returns {Promise.<{
* id: string;
* url: string;
* width: number;
* height: number;
* color: number;
* preview: {
* url: string;
* width: number;
* height: number;
* },
* origin: {
* title: string;
* website: {
* name: string;
* domain: string;
* url: string;
* }
* }
*}[]>}
*/
export function image(query: string, options?: {
safe?: boolean;
additional_params?: object;
}): Promise<{
id: string;
url: string;
width: number;
height: number;
color: number;
preview: {
url: string;
width: number;
height: number;
};
origin: {
title: string;
website: {
name: string;
domain: string;
url: string;
};
};
}[]>;
import KnowledgeGraph = require("./nodes/KnowledgeGraph");
import FeaturedSnippet = require("./nodes/FeaturedSnippet");
import Weather = require("./nodes/Weather");
import Time = require("./nodes/Time");
import Location = require("./nodes/Location");
import Dictionary = require("./nodes/Dictionary");
import Translation = require("./nodes/Translation");
import Converters = require("./nodes/Converters");

View file

@ -0,0 +1,7 @@
export = Converters;
declare class Converters {
constructor($: any);
input: any;
output: any;
formula: any;
}

View file

@ -0,0 +1,14 @@
export = Dictionary;
declare class Dictionary {
constructor($: any);
/** @type {string | null} */
word: string | null;
/** @type {string | null} */
phonetic: string | null;
/** @type {string | null} */
audio: string | null;
/** @type {string[]} */
definitions: string[];
/** @type {string[]} */
examples: string[];
}

View file

@ -0,0 +1,10 @@
export = FeaturedSnippet;
declare class FeaturedSnippet {
constructor($: any);
/** @type {string | null} */
title: string | null;
/** @type {string | null} */
description: string | null;
/** @type {string | null} */
url: string | null;
}

View file

@ -0,0 +1,55 @@
export = KnowledgeGraph;
declare class KnowledgeGraph {
constructor(data: any, $: any);
/** @type {string | null} */
type: string | null;
/** @type {string | null} */
title: string | null;
/** @type {string | null} */
description: string | null;
/** @type {string | null} */
url: string | null;
/** @type {{ title: string; value: string }[]} */
metadata: {
title: string;
value: string;
}[];
/** @type {{ title: string; year: string; }[]} */
books: {
title: string;
year: string;
}[];
/** @type {{ title: string; year: string; }[]} */
tv_shows_and_movies: {
title: string;
year: string;
}[];
ratings: any[];
/** @type {string[]} */
available_on: string[];
/** @type {{ url: string; source: string }[]} */
images: {
url: string;
source: string;
}[];
/** @type {{ title: string; album: string }[]} */
songs: {
title: string;
album: string;
}[];
/** @type {Social[]} */
socials: Social[];
/** @type {string | null} */
demonstration: string | null;
/** @type {string | null} */
lyrics: string | null;
}
declare class Social {
constructor(data: any);
/** @type {string} */
name: string;
/** @type {string} */
url: string;
/** @type {string} */
icon: string;
}

View file

@ -0,0 +1,10 @@
export = Location;
declare class Location {
constructor($: any);
/** @type {string | null} */
title: string | null;
/** @type {string | null} */
distance: string | null;
/** @type {string | null} */
map: string | null;
}

View file

@ -0,0 +1,25 @@
export = OrganicResults;
declare class OrganicResults {
/**
* @returns {{
* title: string;
* description: string;
* url: string;
* is_sponsored: boolean;
* favicons: {
* high_res: string;
* low_res: string;
* }
* }[]}
*/
static parse($: any, parse_ads?: boolean, is_mobile?: boolean): {
title: string;
description: string;
url: string;
is_sponsored: boolean;
favicons: {
high_res: string;
low_res: string;
};
}[];
}

4
node_modules/googlethis/typings/core/nodes/PAA.d.ts generated vendored Normal file
View file

@ -0,0 +1,4 @@
export = PAA;
declare class PAA {
static parse($: any, data: any): string[];
}

7
node_modules/googlethis/typings/core/nodes/PAS.d.ts generated vendored Normal file
View file

@ -0,0 +1,7 @@
export = PAS;
declare class PAS {
static parse($: any): {
title: string;
thumbnail: string;
}[];
}

8
node_modules/googlethis/typings/core/nodes/Time.d.ts generated vendored Normal file
View file

@ -0,0 +1,8 @@
export = Time;
declare class Time {
constructor($: any);
/** @type {string | null} */
hours: string | null;
/** @type {string | null} */
date: string | null;
}

View file

@ -0,0 +1,7 @@
export = TopStories;
declare class TopStories {
static parse($: any): {
description: string;
url: string;
}[];
}

View file

@ -0,0 +1,12 @@
export = Translation;
declare class Translation {
constructor($: any);
/** @type {string | null} */
source_language: string | null;
/** @type {string | null} */
target_language: string | null;
/** @type {string | null} */
source_text: string | null;
/** @type {string | null} */
target_text: string | null;
}

10
node_modules/googlethis/typings/core/nodes/Videos.d.ts generated vendored Normal file
View file

@ -0,0 +1,10 @@
export = Videos;
declare class Videos {
static parse($: any): {
id: string;
url: string;
title: string;
author: string;
duration: string;
}[];
}

View file

@ -0,0 +1,16 @@
export = Weather;
declare class Weather {
constructor($: any);
/** @type {string | null} */
location: string | null;
/** @type {string | null} */
forecast: string | null;
/** @type {string | null} */
precipitation: string | null;
/** @type {string | null} */
humidity: string | null;
/** @type {string | null} */
temperature: string | null;
/** @type {string | null} */
wind: string | null;
}

2
node_modules/googlethis/typings/index.d.ts generated vendored Normal file
View file

@ -0,0 +1,2 @@
declare const _exports: typeof import("./core/main");
export = _exports;

68
node_modules/googlethis/typings/utils/constants.d.ts generated vendored Normal file
View file

@ -0,0 +1,68 @@
export namespace URLS {
const GIS: string;
const GOOGLE: string;
const W_GOOGLE: string;
const GOOGLE_NEWS: string;
const FAVICONKIT: string;
}
export namespace SELECTORS {
const TITLE: string;
const DESCRIPTION: string;
const URL: string;
const DID_YOU_MEAN: string;
const KNO_PANEL_TITLE: string[];
const KNO_PANEL_DESCRIPTION: string;
const KNO_PANEL_URL: string;
const KNO_PANEL_METADATA: string;
const KNO_PANEL_TYPE: string;
const KNO_PANEL_SONG_LYRICS: string;
const KNO_PANEL_AVAILABLE_ON: string;
const KNO_PANEL_IMAGES: string;
const KNO_PANEL_SONGS: string;
const KNO_PANEL_BOOKS: string;
const KNO_PANEL_TV_SHOWS_AND_MOVIES: string;
const KNO_PANEL_FILM_GOOGLEUSERS_RATING: string;
const KNO_PANEL_FILM_RATINGS: string[];
const KNO_PANEL_SOCIALS: string;
const VIDEOS: string;
const FEATURED_SNIPPET_TITLE: string[];
const FEATURED_SNIPPET_DESC: string[];
const FEATURED_SNIPPET_URL: string;
const UNIT_CONVERTER_INPUT: string;
const UNIT_CONVERTER_OUTPUT: string;
const UNIT_CONVERTER_FORMULA: string;
const INPUT_CURRENCY_NAME: string;
const OUTPUT_CURRENCY_NAME: string;
const CURRENCY_CONVERTER_INPUT: string;
const CURRENCY_CONVERTER_OUTPUT: string;
const WEATHER_LOCATION: string;
const WEATHER_FORECAST: string;
const PRECIPITATION: string;
const AIR_HUMIDITY: string;
const TEMPERATURE: string;
const WIND_SPEED: string;
const CURRENT_TIME_HOUR: string;
const CURRENT_TIME_DATE: string;
const LOCATION_TITLE: string;
const LOCATION_DISTANCE: string;
const LOCATION_IMAGE: string;
const GD_WORD: string;
const GD_PHONETIC: string;
const GD_AUDIO: string;
const GD_DEFINITIONS: string;
const GD_EXAMPLES: string;
const TR_SOURCE_LANGUAGE: string;
const TR_TARGET_LANGUAGE: string;
const TR_SOURCE_TEXT: string;
const TR_TARGET_TEXT: string;
const TOP_STORIES_DESCRIPTION: string[];
const TOP_STORIES_URL: string;
const TOP_STORIES_SNIPPET: string;
const TOP_STORIES_WEBSITE: string;
const PAA: string[];
const PASF: string;
const PUBLISHER: string;
const STORY_TITLE: string;
const STORY_IMG: string;
const STORY_TIME: string;
}

48
node_modules/googlethis/typings/utils/utils.d.ts generated vendored Normal file
View file

@ -0,0 +1,48 @@
export class SearchError extends Error {
constructor(message: any, info: any);
info: any;
date: Date;
version: any;
}
/**
* Returns headers with a random user agent.
*
* @param {boolean} is_mobile
* @returns {string}
*/
export function getHeaders(options?: {
mobile: boolean;
}): string;
/**
* Gets a string between two delimiters.
*
* @param {string} data - The data.
* @param {string} start_string - Start string.
* @param {string} end_string - End string.
*
* @returns {string}
*/
export function getStringBetweenStrings(data: string, start_string: string, end_string: string): string;
/**
* Generates a random string with a given length.
* @param {number} length
* @returns {string}
*/
export function generateRandomString(length: number): string;
/**
* Returns a random integer between two values.
*
* @param {number} min
* @param {number} max
*
* @returns {number}
*/
export function getRandomInt(min: number, max: number): number;
/**
* Refines the html.
*
* @param {string} data - Raw html data.
* @param {boolean} parse_ads - Whether to parse ads or not.
* @returns {string}
*/
export function refineData(data: string, parse_ads?: boolean, is_mobile?: boolean): string;