From 7389f85ef2c031c8b15410fff93a44ac08671260 Mon Sep 17 00:00:00 2001 From: the-superpirate Date: Fri, 8 Jan 2021 23:08:47 +0300 Subject: [PATCH] - Send Pylon to golden - Move fancy naming to Hub - Adopt translations - Fix long authors list in bot - Support CJK in scitech - Fixed pylon bug with decoding improper unicode ch... GitOrigin-RevId: 74f73c44f749a71cb65dd5ddd3416f32a83d329f --- images/production/BUILD.bazel | 1 + nexus/README.md | 2 +- nexus/pylon/BUILD.bazel | 34 +++ nexus/pylon/README.md | 21 ++ nexus/pylon/__init__.py | 0 nexus/pylon/cli.py | 81 +++++++ nexus/pylon/client.py | 98 +++++++++ nexus/pylon/exceptions.py | 39 ++++ nexus/pylon/pdftools/__init__.py | 9 + nexus/pylon/proto/BUILD.bazel | 19 ++ nexus/pylon/proto/file.proto | 18 ++ nexus/pylon/proto/file_pb2.py | 165 ++++++++++++++ nexus/pylon/sources/__init__.py | 19 ++ nexus/pylon/sources/base.py | 202 ++++++++++++++++++ nexus/pylon/sources/libgen_doi.py | 34 +++ nexus/pylon/sources/libgen_md5.py | 37 ++++ nexus/pylon/sources/libgen_new.py | 52 +++++ nexus/pylon/sources/scihub.py | 49 +++++ nexus/pylon/sources/specific/__init__.py | 20 ++ nexus/pylon/sources/specific/biorxiv.py | 19 ++ nexus/pylon/sources/specific/lancet.py | 24 +++ nexus/pylon/sources/specific/nejm.py | 21 ++ .../pylon/sources/specific/research_square.py | 30 +++ nexus/summa/configs/config.yaml | 2 +- nexus/summa/schema/scitech.yaml | 10 +- nexus/translations/translations.yaml | 141 +++++++++++- rules/rust/Cargo.toml | 3 +- rules/rust/cargo/BUILD.bazel | 11 +- rules/rust/crates.bzl | 153 +++++++++++++ rules/rust/remote/BUILD.bstr-0.2.14.bazel | 81 +++++++ rules/rust/remote/BUILD.cast-0.2.3.bazel | 87 ++++++++ rules/rust/remote/BUILD.criterion-0.3.3.bazel | 78 +++++++ .../remote/BUILD.criterion-plot-0.4.3.bazel | 55 +++++ rules/rust/remote/BUILD.csv-1.1.5.bazel | 134 ++++++++++++ rules/rust/remote/BUILD.csv-core-0.1.10.bazel | 57 +++++ rules/rust/remote/BUILD.half-1.6.0.bazel | 57 +++++ rules/rust/remote/BUILD.js-sys-0.3.46.bazel | 58 +++++ rules/rust/remote/BUILD.oorandom-11.1.3.bazel | 53 +++++ rules/rust/remote/BUILD.plotters-0.2.15.bazel | 105 +++++++++ .../remote/BUILD.regex-automata-0.1.9.bazel | 56 +++++ rules/rust/remote/BUILD.same-file-1.0.6.bazel | 57 +++++ .../rust/remote/BUILD.serde_cbor-0.11.1.bazel | 77 +++++++ .../remote/BUILD.tinytemplate-1.2.0.bazel | 57 +++++ rules/rust/remote/BUILD.walkdir-2.3.1.bazel | 54 +++++ rules/rust/remote/BUILD.web-sys-0.3.46.bazel | 67 ++++++ .../rust/remote/BUILD.winapi-util-0.1.5.bazel | 53 +++++ 46 files changed, 2490 insertions(+), 10 deletions(-) create mode 100644 nexus/pylon/BUILD.bazel create mode 100644 nexus/pylon/README.md create mode 100644 nexus/pylon/__init__.py create mode 100644 nexus/pylon/cli.py create mode 100644 nexus/pylon/client.py create mode 100644 nexus/pylon/exceptions.py create mode 100644 nexus/pylon/pdftools/__init__.py create mode 100644 nexus/pylon/proto/BUILD.bazel create mode 100644 nexus/pylon/proto/file.proto create mode 100644 nexus/pylon/proto/file_pb2.py create mode 100644 nexus/pylon/sources/__init__.py create mode 100644 nexus/pylon/sources/base.py create mode 100644 nexus/pylon/sources/libgen_doi.py create mode 100644 nexus/pylon/sources/libgen_md5.py create mode 100644 nexus/pylon/sources/libgen_new.py create mode 100644 nexus/pylon/sources/scihub.py create mode 100644 nexus/pylon/sources/specific/__init__.py create mode 100644 nexus/pylon/sources/specific/biorxiv.py create mode 100644 nexus/pylon/sources/specific/lancet.py create mode 100644 nexus/pylon/sources/specific/nejm.py create mode 100644 nexus/pylon/sources/specific/research_square.py create mode 100644 rules/rust/remote/BUILD.bstr-0.2.14.bazel create mode 100644 rules/rust/remote/BUILD.cast-0.2.3.bazel create mode 100644 rules/rust/remote/BUILD.criterion-0.3.3.bazel create mode 100644 rules/rust/remote/BUILD.criterion-plot-0.4.3.bazel create mode 100644 rules/rust/remote/BUILD.csv-1.1.5.bazel create mode 100644 rules/rust/remote/BUILD.csv-core-0.1.10.bazel create mode 100644 rules/rust/remote/BUILD.half-1.6.0.bazel create mode 100644 rules/rust/remote/BUILD.js-sys-0.3.46.bazel create mode 100644 rules/rust/remote/BUILD.oorandom-11.1.3.bazel create mode 100644 rules/rust/remote/BUILD.plotters-0.2.15.bazel create mode 100644 rules/rust/remote/BUILD.regex-automata-0.1.9.bazel create mode 100644 rules/rust/remote/BUILD.same-file-1.0.6.bazel create mode 100644 rules/rust/remote/BUILD.serde_cbor-0.11.1.bazel create mode 100644 rules/rust/remote/BUILD.tinytemplate-1.2.0.bazel create mode 100644 rules/rust/remote/BUILD.walkdir-2.3.1.bazel create mode 100644 rules/rust/remote/BUILD.web-sys-0.3.46.bazel create mode 100644 rules/rust/remote/BUILD.winapi-util-0.1.5.bazel diff --git a/images/production/BUILD.bazel b/images/production/BUILD.bazel index 32561f9..ad8d8f3 100644 --- a/images/production/BUILD.bazel +++ b/images/production/BUILD.bazel @@ -9,6 +9,7 @@ download_pkgs( image_tar = "//images:base-image.tar", packages = [ "bash", + "ca-certificates", "libev4", "libgomp1", "libgoogle-perftools-dev", diff --git a/nexus/README.md b/nexus/README.md index df555d0..fb6f32d 100644 --- a/nexus/README.md +++ b/nexus/README.md @@ -11,7 +11,7 @@ - ✅ [`models`](models) - shared Protobuf models - ✅ [`nlptools`](nlptools) - text routines - ✅ [`pipe`](pipe) - processing pipeline based on Kafka -- 🛑 `pylon` - smart proxy for downloading files from the Internet/IPFS +- ✅ [`pylon`](pylon) - smart client for downloading files from the Internet/IPFS - ✅ [`summa`](summa) - scripts for setting Summa - ✅ [`translations`](translations) - text translations used in `bot` and `hub` - 🛑 `views` - shared views for [`models`](models) diff --git a/nexus/pylon/BUILD.bazel b/nexus/pylon/BUILD.bazel new file mode 100644 index 0000000..8bf7e05 --- /dev/null +++ b/nexus/pylon/BUILD.bazel @@ -0,0 +1,34 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("@pip_modules_external//:requirements.bzl", "requirement") + +py_library( + name = "pylon", + srcs = glob(["**/*.py"]), + visibility = ["//visibility:public"], + deps = [ + requirement("aiodns"), + requirement("aiohttp"), + requirement("aiohttp_socks"), + requirement("brotli"), + requirement("cchardet"), + requirement("orjson"), + requirement("pdfminer.six"), + requirement("python-socks"), + requirement("tenacity"), + requirement("aiokit"), + "//library/logging", + "//nexus/pylon/proto:pylon_proto_py", + ], +) + +py_binary( + name = "cli", + srcs = ["cli.py"], + main = "cli.py", + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + requirement("fire"), + ":pylon", + ], +) diff --git a/nexus/pylon/README.md b/nexus/pylon/README.md new file mode 100644 index 0000000..41e2f9f --- /dev/null +++ b/nexus/pylon/README.md @@ -0,0 +1,21 @@ +# Nexus Pylon + +`Pylon` is a downloader for scientific publications. +- Look articles by DOI, MD5 or IPFS hashes +- Validates downloaded items +- Streams data by chunks +- GRPC-ready + +## Nexus Pylon CLI + +Casual download +```bash +bazel run -c opt cli -- doi 10.1056/NEJMoa2033700 --output article.pdf +``` + +Download with proxy +```bash +bazel run -c opt cli -- md5 278C3A72B7B04717361501B8642857DF \ + --output file.pdf \ + --proxy socks5://127.0.0.1:9050 +``` diff --git a/nexus/pylon/__init__.py b/nexus/pylon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nexus/pylon/cli.py b/nexus/pylon/cli.py new file mode 100644 index 0000000..585696e --- /dev/null +++ b/nexus/pylon/cli.py @@ -0,0 +1,81 @@ +import os +from typing import Optional + +import fire +from aiokit.utils import sync_fu +from nexus.pylon.client import ( + DownloadError, + PylonClient, +) +from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb + + +def resolve_path(filepath): + if os.path.isabs(filepath): + return filepath + cwd = os.environ.get('BUILD_WORKING_DIRECTORY', os.getcwd()) + filepath = os.path.join(cwd, filepath) + return filepath + + +async def fetch( + iter, + output: str, +): + collected = bytes() + try: + last_len = 0 + last_source = '' + async for resp in iter: + if resp.HasField('status'): + if resp.status == FileResponsePb.Status.BEGIN_TRANSMISSION: + print(f'Started transmission from {resp.source}...', end='\r') + last_len = 0 + last_source = resp.source + collected = bytes() + elif resp.HasField('chunk'): + if len(collected) - last_len > 1024 * 100: + print(f'Loaded {len(collected)} bytes from {resp.source}', end='\r') + last_len = len(collected) + last_source = resp.source + collected += resp.chunk.content + with open(resolve_path(output), 'wb') as f: + print(f'Completed! Loaded {len(collected)} bytes from {last_source}') + f.write(collected) + except DownloadError: + print('File not found') + + +async def fetch_by_md5( + md5: str, + output: str, + proxy: Optional[str] = None, + resolve_proxy: Optional[str] = None, +): + if proxy and not resolve_proxy: + resolve_proxy = proxy + p = PylonClient(proxy=proxy, resolve_proxy=resolve_proxy) + return await fetch(iter=p.by_md5(md5=md5), output=output) + + +async def fetch_by_doi( + doi: str, + output: str, + proxy: Optional[str] = None, + resolve_proxy: Optional[str] = None, +): + if proxy and not resolve_proxy: + resolve_proxy = proxy + p = PylonClient(proxy=proxy, resolve_proxy=resolve_proxy) + return await fetch(iter=p.by_doi(doi=doi), output=output) + + +def main(): + fire.Fire({ + 'doi': sync_fu(fetch_by_doi), + 'md5': sync_fu(fetch_by_md5), + }) + + +if __name__ == '__main__': + main() diff --git a/nexus/pylon/client.py b/nexus/pylon/client.py new file mode 100644 index 0000000..23c15e3 --- /dev/null +++ b/nexus/pylon/client.py @@ -0,0 +1,98 @@ +import asyncio +from typing import ( + AsyncIterable, + Iterable, + Optional, +) + +import aiohttp +import aiohttp.client_exceptions +from aiohttp_socks import ( + ProxyConnectionError, + ProxyError, +) +from aiokit import AioThing +from library.logging import error_log +from nexus.pylon.exceptions import ( + DownloadError, + NotFoundError, +) +from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb +from nexus.pylon.sources import ( + BaseSource, + LibgenDoiSource, + LibgenMd5Source, + LibraryLolSource, + SciHubSeSource, + SciHubTwSource, +) +from nexus.pylon.sources.specific import get_specific_sources_for_doi +from python_socks import ProxyTimeoutError + + +class PylonClient(AioThing): + def __init__(self, proxy: Optional[str] = None, resolve_proxy: Optional[str] = None): + super().__init__() + self.proxy = proxy + self.resolve_proxy = resolve_proxy + + async def by_doi( + self, + doi: str, + md5: Optional[str] = None, + ) -> AsyncIterable[FileResponsePb]: + sources = [] + sources.extend(get_specific_sources_for_doi(doi, proxy=self.proxy, resolve_proxy=self.resolve_proxy)) + sources.extend([ + SciHubSeSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), + SciHubTwSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), + LibgenDoiSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), + ]) + sources = filter(lambda x: x.is_enabled, sources) + async for resp in self.download(sources=sources): + yield resp + + async def by_md5( + self, + md5: str, + ) -> AsyncIterable[FileResponsePb]: + sources = filter(lambda x: x.is_enabled, [ + LibraryLolSource(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), + LibgenMd5Source(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy), + ]) + async for resp in self.download(sources=sources): + yield resp + + async def download_source(self, source) -> AsyncIterable[FileResponsePb]: + try: + yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url) + async for prepared_request in source.resolve(): + async for resp in source.execute_prepared_request(prepared_request=prepared_request): + yield resp + return + raise DownloadError(error='not_found', source=str(source)) + except ( + aiohttp.client_exceptions.ClientConnectionError, + aiohttp.client_exceptions.ClientPayloadError, + aiohttp.client_exceptions.ClientResponseError, + aiohttp.client_exceptions.TooManyRedirects, + asyncio.TimeoutError, + ProxyConnectionError, + ProxyTimeoutError, + ProxyError, + ) as e: + raise DownloadError(nested_error=str(e), nested_error_cls=e.__class__.__name__) + + async def download(self, sources: Iterable[BaseSource]) -> AsyncIterable[FileResponsePb]: + for source in sources: + try: + await source.start() + async for resp in self.download_source(source): + yield resp + return + except DownloadError as e: + error_log(e) + continue + finally: + await source.stop() + raise NotFoundError() diff --git a/nexus/pylon/exceptions.py b/nexus/pylon/exceptions.py new file mode 100644 index 0000000..d5aaf32 --- /dev/null +++ b/nexus/pylon/exceptions.py @@ -0,0 +1,39 @@ +import logging + +from izihawa_utils.exceptions import BaseError + + +class DownloadError(BaseError): + level = logging.WARNING + code = 'download_error' + + +class IncorrectMD5Error(DownloadError): + code = 'incorrect_md5_error' + + +class UnresolvableSourceError(DownloadError): + level = logging.WARNING + code = 'unresolvable_source_error' + + +class UnavailableSourcesError(DownloadError): + code = 'no_available_sources_error' + + +class NotFoundError(DownloadError): + level = logging.WARNING + code = 'not_found_error' + + +class RegexNotFoundError(DownloadError): + level = logging.WARNING + code = 'regex_not_found_error' + + +class BadResponseError(DownloadError): + code = 'bad_response_error' + + +class InvalidDocumentForDownload(DownloadError): + code = 'invalid_document_for_download' diff --git a/nexus/pylon/pdftools/__init__.py b/nexus/pylon/pdftools/__init__.py new file mode 100644 index 0000000..2d18782 --- /dev/null +++ b/nexus/pylon/pdftools/__init__.py @@ -0,0 +1,9 @@ +from pdfminer.high_level import extract_text + + +def is_pdf(path_to_file): + try: + extract_text(path_to_file, maxpages=1) + return True + except Exception: + return False diff --git a/nexus/pylon/proto/BUILD.bazel b/nexus/pylon/proto/BUILD.bazel new file mode 100644 index 0000000..69a975e --- /dev/null +++ b/nexus/pylon/proto/BUILD.bazel @@ -0,0 +1,19 @@ +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +package(default_visibility = ["//visibility:public"]) + +proto_library( + name = "pylon_proto", + srcs = glob([ + "*.proto", + ]), + deps = [ + "@com_google_protobuf//:wrappers_proto", + ], +) + +py_proto_library( + name = "pylon_proto_py", + deps = [":pylon_proto"], +) diff --git a/nexus/pylon/proto/file.proto b/nexus/pylon/proto/file.proto new file mode 100644 index 0000000..68a1093 --- /dev/null +++ b/nexus/pylon/proto/file.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; +package nexus.pylon.proto; + +message Chunk { + bytes content = 1; +} + +message FileResponse { + enum Status { + RESOLVING = 0; + BEGIN_TRANSMISSION = 1; + } + oneof response { + Chunk chunk = 1; + Status status = 2; + } + string source = 3; +} diff --git a/nexus/pylon/proto/file_pb2.py b/nexus/pylon/proto/file_pb2.py new file mode 100644 index 0000000..57eed32 --- /dev/null +++ b/nexus/pylon/proto/file_pb2.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: nexus/pylon/proto/file.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='nexus/pylon/proto/file.proto', + package='nexus.pylon.proto', + syntax='proto3', + serialized_options=None, + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x1cnexus/pylon/proto/file.proto\x12\x11nexus.pylon.proto\"\x18\n\x05\x43hunk\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\"\xc0\x01\n\x0c\x46ileResponse\x12)\n\x05\x63hunk\x18\x01 \x01(\x0b\x32\x18.nexus.pylon.proto.ChunkH\x00\x12\x38\n\x06status\x18\x02 \x01(\x0e\x32&.nexus.pylon.proto.FileResponse.StatusH\x00\x12\x0e\n\x06source\x18\x03 \x01(\t\"/\n\x06Status\x12\r\n\tRESOLVING\x10\x00\x12\x16\n\x12\x42\x45GIN_TRANSMISSION\x10\x01\x42\n\n\x08responseb\x06proto3' +) + + + +_FILERESPONSE_STATUS = _descriptor.EnumDescriptor( + name='Status', + full_name='nexus.pylon.proto.FileResponse.Status', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='RESOLVING', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='BEGIN_TRANSMISSION', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + serialized_start=211, + serialized_end=258, +) +_sym_db.RegisterEnumDescriptor(_FILERESPONSE_STATUS) + + +_CHUNK = _descriptor.Descriptor( + name='Chunk', + full_name='nexus.pylon.proto.Chunk', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='content', full_name='nexus.pylon.proto.Chunk.content', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=51, + serialized_end=75, +) + + +_FILERESPONSE = _descriptor.Descriptor( + name='FileResponse', + full_name='nexus.pylon.proto.FileResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='chunk', full_name='nexus.pylon.proto.FileResponse.chunk', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='status', full_name='nexus.pylon.proto.FileResponse.status', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='source', full_name='nexus.pylon.proto.FileResponse.source', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _FILERESPONSE_STATUS, + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='response', full_name='nexus.pylon.proto.FileResponse.response', + index=0, containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[]), + ], + serialized_start=78, + serialized_end=270, +) + +_FILERESPONSE.fields_by_name['chunk'].message_type = _CHUNK +_FILERESPONSE.fields_by_name['status'].enum_type = _FILERESPONSE_STATUS +_FILERESPONSE_STATUS.containing_type = _FILERESPONSE +_FILERESPONSE.oneofs_by_name['response'].fields.append( + _FILERESPONSE.fields_by_name['chunk']) +_FILERESPONSE.fields_by_name['chunk'].containing_oneof = _FILERESPONSE.oneofs_by_name['response'] +_FILERESPONSE.oneofs_by_name['response'].fields.append( + _FILERESPONSE.fields_by_name['status']) +_FILERESPONSE.fields_by_name['status'].containing_oneof = _FILERESPONSE.oneofs_by_name['response'] +DESCRIPTOR.message_types_by_name['Chunk'] = _CHUNK +DESCRIPTOR.message_types_by_name['FileResponse'] = _FILERESPONSE +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Chunk = _reflection.GeneratedProtocolMessageType('Chunk', (_message.Message,), { + 'DESCRIPTOR' : _CHUNK, + '__module__' : 'nexus.pylon.proto.file_pb2' + # @@protoc_insertion_point(class_scope:nexus.pylon.proto.Chunk) + }) +_sym_db.RegisterMessage(Chunk) + +FileResponse = _reflection.GeneratedProtocolMessageType('FileResponse', (_message.Message,), { + 'DESCRIPTOR' : _FILERESPONSE, + '__module__' : 'nexus.pylon.proto.file_pb2' + # @@protoc_insertion_point(class_scope:nexus.pylon.proto.FileResponse) + }) +_sym_db.RegisterMessage(FileResponse) + + +# @@protoc_insertion_point(module_scope) diff --git a/nexus/pylon/sources/__init__.py b/nexus/pylon/sources/__init__.py new file mode 100644 index 0000000..e9df95a --- /dev/null +++ b/nexus/pylon/sources/__init__.py @@ -0,0 +1,19 @@ +from .base import BaseSource +from .libgen_doi import LibgenDoiSource +from .libgen_md5 import LibgenMd5Source +from .libgen_new import ( + LibraryLolSource, +) +from .scihub import ( + SciHubSeSource, + SciHubTwSource, +) + +__all__ = [ + 'BaseSource', + 'LibgenDoiSource', + 'LibgenMd5Source', + 'LibraryLolSource', + 'SciHubSeSource', + 'SciHubTwSource', +] diff --git a/nexus/pylon/sources/base.py b/nexus/pylon/sources/base.py new file mode 100644 index 0000000..fefa408 --- /dev/null +++ b/nexus/pylon/sources/base.py @@ -0,0 +1,202 @@ +import hashlib +import random +from typing import ( + AsyncIterable, + Optional, +) + +import aiohttp +import aiohttp.client_exceptions +from aiohttp_socks import ( + ProxyConnector, + ProxyError, +) +from aiokit import AioThing +from nexus.pylon.exceptions import ( + BadResponseError, + IncorrectMD5Error, + NotFoundError, +) +from nexus.pylon.pdftools import is_pdf +from nexus.pylon.proto.file_pb2 import Chunk as ChunkPb +from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb +from python_socks import ProxyTimeoutError +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, +) + +DEFAULT_USER_AGENT = 'PylonBot/1.0 (Linux x86_64) PylonBot/1.0.0' + + +class PreparedRequest: + def __init__( + self, + method: str, + url: str, + headers: dict = None, + params: dict = None, + cookies: dict = None, + ssl: bool = True, + ): + self.method = method + self.url = url + self.headers = { + 'User-Agent': DEFAULT_USER_AGENT, + } + if headers: + self.headers.update(headers) + self.params = params + self.cookies = cookies + self.ssl = ssl + + def __repr__(self): + return f'{self.method} {self.url} {self.headers} {self.params}' + + def __str__(self): + return repr(self) + + +class BaseValidator: + def update(self, chunk: bytes): + pass + + def validate(self): + pass + + +class Md5Validator(BaseValidator): + def __init__(self, md5: str): + self.md5 = md5 + self.v = hashlib.md5() + + def update(self, chunk: bytes): + self.v.update(chunk) + + def validate(self): + digest = self.v.hexdigest() + if self.md5.lower() != self.v.hexdigest().lower(): + raise IncorrectMD5Error(requested_md5=self.md5, downloaded_md5=digest) + + +class DoiValidator(BaseValidator): + def __init__(self, doi: str): + self.doi = doi + self.file = bytes() + + def update(self, chunk): + self.file += chunk + + def validate(self): + return is_pdf(self.file) + + +class BaseSource(AioThing): + allowed_content_type = None + base_url = None + is_enabled = True + resolve_timeout = None + ssl = True + timeout = None + use_proxy = None + + def __init__(self, proxy: str = None, resolve_proxy: str = None): + super().__init__() + self.proxy = proxy + self.resolve_proxy = resolve_proxy + + def get_proxy(self): + if self.proxy and self.use_proxy is not False: + return ProxyConnector.from_url(self.proxy, verify_ssl=self.ssl) + return aiohttp.TCPConnector(verify_ssl=self.ssl) + + def get_resolve_proxy(self): + if self.resolve_proxy and self.use_proxy is not False: + return ProxyConnector.from_url(self.resolve_proxy, verify_ssl=self.ssl) + return aiohttp.TCPConnector(verify_ssl=self.ssl) + + def get_session(self): + return aiohttp.ClientSession(connector=self.get_proxy()) + + def get_resolve_session(self): + return aiohttp.ClientSession(connector=self.get_resolve_proxy()) + + def resolve(self) -> AsyncIterable[PreparedRequest]: + raise NotImplementedError("`resolve` for BaseSource is not implemented") + + def get_validator(self): + return BaseValidator() + + @retry( + reraise=True, + stop=stop_after_attempt(3), + retry=retry_if_exception_type((ProxyError, aiohttp.client_exceptions.ClientPayloadError, ProxyTimeoutError)), + ) + async def execute_prepared_request(self, prepared_request: PreparedRequest): + async with self.get_session() as session: + async with session.request( + method=prepared_request.method, + url=prepared_request.url, + timeout=self.timeout, + headers=prepared_request.headers, + cookies=prepared_request.cookies, + params=prepared_request.params, + ssl=prepared_request.ssl, + ) as resp: + if resp.status == 404: + raise NotFoundError(url=prepared_request.url) + elif ( + resp.status != 200 + or ( + self.allowed_content_type + and resp.headers.get('Content-Type', '').lower() not in self.allowed_content_type + ) + ): + raise BadResponseError( + request_headers=prepared_request.headers, + url=prepared_request.url, + status=resp.status, + headers=str(resp.headers), + ) + file_validator = self.get_validator() + # Randomness is required due to annoying bug of when separators + # (\r\n) are splitted to different chunks + # https://github.com/aio-libs/aiohttp/issues/4677 + yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_request.url) + async for content in resp.content.iter_chunked(1024 * 100 + random.randint(-1024, 1024)): + file_validator.update(content) + yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_request.url) + file_validator.validate() + + +class Md5Source(BaseSource): + def __init__( + self, + md5: str, + proxy: Optional[str] = None, + resolve_proxy: Optional[str] = None, + ): + super().__init__(proxy=proxy, resolve_proxy=resolve_proxy) + self.md5 = md5 + + def get_validator(self): + return Md5Validator(self.md5) + + +class DoiSource(BaseSource): + def __init__( + self, + doi: str, + md5: Optional[str] = None, + proxy: Optional[str] = None, + resolve_proxy: Optional[str] = None, + ): + super().__init__(proxy=proxy, resolve_proxy=resolve_proxy) + self.doi = doi + self.md5 = md5 + + def get_validator(self): + if self.md5: + return Md5Validator(self.md5) + return DoiValidator(self.doi) diff --git a/nexus/pylon/sources/libgen_doi.py b/nexus/pylon/sources/libgen_doi.py new file mode 100644 index 0000000..765b167 --- /dev/null +++ b/nexus/pylon/sources/libgen_doi.py @@ -0,0 +1,34 @@ +import re +from typing import AsyncIterable + +from library.logging import error_log +from nexus.pylon.exceptions import RegexNotFoundError + +from .base import ( + DoiSource, + PreparedRequest, +) + + +class LibgenDoiSource(DoiSource): + base_url = 'http://libgen.gs' + resolve_timeout = 10 + + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/scimag/ads.php?doi={self.doi}' + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + downloaded_page_bytes = await resp.read() + downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace') + match = re.search( + 'https?://.*/scimag/get\\.php\\?doi=.*&key=[A-Za-z0-9]+', + downloaded_page, + re.IGNORECASE, + ) + if match: + yield PreparedRequest(method='get', url=match.group()) + else: + error_log(RegexNotFoundError(url=url)) diff --git a/nexus/pylon/sources/libgen_md5.py b/nexus/pylon/sources/libgen_md5.py new file mode 100644 index 0000000..ececc91 --- /dev/null +++ b/nexus/pylon/sources/libgen_md5.py @@ -0,0 +1,37 @@ +import re +from typing import AsyncIterable + +from .base import ( + Md5Source, + PreparedRequest, +) + + +class LibgenMd5Source(Md5Source): + base_url = 'http://libgen.gs' + resolve_timeout = 10 + + async def resolve_lg(self, session, url): + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + downloaded_page_fiction = await resp.text() + match = re.search( + 'https?://.*/get\\.php\\?md5=.*&key=[A-Za-z0-9]+', + downloaded_page_fiction, + re.IGNORECASE, + ) + if match: + return PreparedRequest(method='get', url=match.group()) + + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/ads.php?md5={self.md5}' + result = await self.resolve_lg(session, url) + if result: + yield result + url = f'{self.base_url}/foreignfiction/ads.php?md5={self.md5}' + result = await self.resolve_lg(session, url) + if result: + yield result diff --git a/nexus/pylon/sources/libgen_new.py b/nexus/pylon/sources/libgen_new.py new file mode 100644 index 0000000..050fe43 --- /dev/null +++ b/nexus/pylon/sources/libgen_new.py @@ -0,0 +1,52 @@ +import re +from typing import AsyncIterable + +from library.logging import error_log +from nexus.pylon.exceptions import RegexNotFoundError + +from .base import ( + Md5Source, + PreparedRequest, +) + + +class LibgenNewSource(Md5Source): + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/main/{self.md5.upper()}' + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + downloaded_page = await resp.text() + match_ipfs = re.search( + 'https://ipfs.io/ipfs/[A-Za-z0-9]+', + downloaded_page, + re.IGNORECASE, + ) + if match_ipfs: + yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl) + match_cf = re.search( + 'https://cloudflare-ipfs.com/ipfs/[A-Za-z0-9]+', + downloaded_page, + re.IGNORECASE, + ) + if match_cf: + yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl) + match_infura = re.search( + 'https://ipfs.infura.io/ipfs/[A-Za-z0-9]+', + downloaded_page, + re.IGNORECASE, + ) + if match_infura: + yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl) + + if not match_cf or not match_infura or not match_ipfs: + error_log(RegexNotFoundError(url=url)) + + +class LibraryLolSource(LibgenNewSource): + base_url = 'http://library.lol' + resolve_timeout = 10 + ssl = False + timeout = 20 diff --git a/nexus/pylon/sources/scihub.py b/nexus/pylon/sources/scihub.py new file mode 100644 index 0000000..60fd217 --- /dev/null +++ b/nexus/pylon/sources/scihub.py @@ -0,0 +1,49 @@ +import re +from typing import AsyncIterable + +from library.logging import error_log +from nexus.pylon.exceptions import RegexNotFoundError + +from .base import ( + DoiSource, + PreparedRequest, +) + + +class SciHubSource(DoiSource): + allowed_content_type = { + 'application/octet-stream', + 'application/pdf', + 'application/pdf;charset=utf-8', + } + base_url = None + ssl = False + + async def resolve(self, timeout=None) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/{self.doi}' + async with session.get( + url, + timeout=timeout or self.timeout + ) as resp: + # Sometimes sci-hub returns file + if resp.headers.get('Content-Type') == 'application/pdf': + yield PreparedRequest(method='get', url=url) + downloaded_page_bytes = await resp.read() + downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace') + match = re.search('(?:https?:)?//.*\\?download=true', downloaded_page, re.IGNORECASE) + if match: + url = match.group() + if url.startswith('//'): + url = 'http:' + url + yield PreparedRequest(method='get', url=url) + else: + error_log(RegexNotFoundError(url=url)) + + +class SciHubSeSource(SciHubSource): + base_url = 'https://sci-hub.se' + + +class SciHubTwSource(SciHubSource): + base_url = 'https://sci-hub.tw' diff --git a/nexus/pylon/sources/specific/__init__.py b/nexus/pylon/sources/specific/__init__.py new file mode 100644 index 0000000..9fd1600 --- /dev/null +++ b/nexus/pylon/sources/specific/__init__.py @@ -0,0 +1,20 @@ +from typing import Iterable + +from ..base import DoiSource +from .biorxiv import BiorxivSource +from .lancet import LancetSource +from .nejm import NejmSource +from .research_square import ResearchSquareSource + +paper_sources = { + '10.1016': [LancetSource], + '10.1056': [NejmSource], + '10.1101': [BiorxivSource], + '10.21203': [ResearchSquareSource], +} + + +def get_specific_sources_for_doi(doi: str, **kwargs) -> Iterable[DoiSource]: + source_clses = paper_sources.get(doi.split('/')[0], []) + source_clses = list(map(lambda cls: cls(doi, **kwargs), source_clses)) + return source_clses diff --git a/nexus/pylon/sources/specific/biorxiv.py b/nexus/pylon/sources/specific/biorxiv.py new file mode 100644 index 0000000..1190f1b --- /dev/null +++ b/nexus/pylon/sources/specific/biorxiv.py @@ -0,0 +1,19 @@ +from typing import AsyncIterable + +from nexus.pylon.sources.base import ( + DoiSource, + PreparedRequest, +) + + +class BiorxivSource(DoiSource): + base_url = 'https://dx.doi.org' + + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/{self.doi}' + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf') diff --git a/nexus/pylon/sources/specific/lancet.py b/nexus/pylon/sources/specific/lancet.py new file mode 100644 index 0000000..95ef527 --- /dev/null +++ b/nexus/pylon/sources/specific/lancet.py @@ -0,0 +1,24 @@ +from typing import AsyncIterable + +from nexus.pylon.sources.base import ( + DoiSource, + PreparedRequest, +) + + +class LancetSource(DoiSource): + base_url = 'https://www.thelancet.com' + resolve_timeout = 10 + use_proxy = False + + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + splitted_doi = self.doi.split("/", maxsplit=1) + if len(splitted_doi) < 2: + return + url = f'{self.base_url}/action/showPdf?pii={splitted_doi[1].upper()}' + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url)) diff --git a/nexus/pylon/sources/specific/nejm.py b/nexus/pylon/sources/specific/nejm.py new file mode 100644 index 0000000..4c86cc1 --- /dev/null +++ b/nexus/pylon/sources/specific/nejm.py @@ -0,0 +1,21 @@ +from typing import AsyncIterable + +from nexus.pylon.sources.base import ( + DoiSource, + PreparedRequest, +) + + +class NejmSource(DoiSource): + base_url = 'https://www.nejm.org' + resolve_timeout = 10 + use_proxy = False + + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/doi/pdf/{self.doi}' + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url)) diff --git a/nexus/pylon/sources/specific/research_square.py b/nexus/pylon/sources/specific/research_square.py new file mode 100644 index 0000000..e83d76c --- /dev/null +++ b/nexus/pylon/sources/specific/research_square.py @@ -0,0 +1,30 @@ +import re +from typing import AsyncIterable + +from nexus.pylon.exceptions import RegexNotFoundError + +from nexus.pylon.sources.base import ( + DoiSource, + PreparedRequest, +) + + +class ResearchSquareSource(DoiSource): + base_url = 'https://dx.doi.org' + + async def resolve(self) -> AsyncIterable[PreparedRequest]: + async with self.get_resolve_session() as session: + url = f'{self.base_url}/{self.doi}' + async with session.get( + url, + timeout=self.resolve_timeout + ) as resp: + download_page = await resp.text() + match = re.search( + r'\"(https://www\.researchsquare\.com/article/[^\"]+\.pdf)\"', + download_page, + re.IGNORECASE, + ) + if not match: + raise RegexNotFoundError(url=url) + yield PreparedRequest(method='get', url=match.group(1)) diff --git a/nexus/summa/configs/config.yaml b/nexus/summa/configs/config.yaml index db3eb25..aeb2765 100644 --- a/nexus/summa/configs/config.yaml +++ b/nexus/summa/configs/config.yaml @@ -7,7 +7,7 @@ http: workers: 4 log_path: /var/log/summa/{{ ENV_TYPE }} search_engine: - data_path: /summa + data_path: /summa/20210108 default_page_size: 5 timeout_secs: 5 writer_memory_mb: 1024 diff --git a/nexus/summa/schema/scitech.yaml b/nexus/summa/schema/scitech.yaml index ca925d2..1ac788d 100644 --- a/nexus/summa/schema/scitech.yaml +++ b/nexus/summa/schema/scitech.yaml @@ -17,7 +17,7 @@ schema: options: indexing: record: position - tokenizer: default + tokenizer: summa stored: true - name: cu type: text @@ -34,7 +34,7 @@ schema: options: indexing: record: position - tokenizer: default + tokenizer: summa stored: true - name: doi type: text @@ -120,21 +120,21 @@ schema: options: indexing: record: position - tokenizer: default + tokenizer: summa stored: true - name: tags type: text options: indexing: record: position - tokenizer: default + tokenizer: summa stored: true - name: title type: text options: indexing: record: position - tokenizer: default + tokenizer: summa stored: true - name: updated_at type: i64 diff --git a/nexus/translations/translations.yaml b/nexus/translations/translations.yaml index 2a7625d..85f027c 100644 --- a/nexus/translations/translations.yaml +++ b/nexus/translations/translations.yaml @@ -24,7 +24,7 @@ en: You can directly feedback us by /contact COPYRIGHT_DESCRIPTION: | - If you with to make a copyright infringement complaint: + If you wish to make a copyright infringement complaint: - Start your message with /copyright, then add the new line - Paste view links of items you are claiming about. View link is an item @@ -160,6 +160,145 @@ en: UPLOADED_TO_TELEGRAM: uploaded to Telegram... VIEWS_CANNOT_BE_SHARED: Do your own search, commands cannot be shared. YEAR: Year +es: + ABSTRACT: Resumen + ALREADY_DOWNLOADING: | + Encontré este libro en Internet. + Por favor, ten paciencia, te lo enviaré cuando esté listo. + AUTHORS: Autores + BANNED: | + Has sido baneado hasta la(s) **{datetime} UTC**. + Razón: **{reason}** + BANNED_FOR_SECONDS: | + Has sido baneado por **{seconds} segundo(s)** + Razón: **{reason}** + BAN_MESSAGE_TOO_MANY_REQUESTS: Demasiadas peticiones + CLOSE: Cerrar + CONTACT: | + **Actualizaciones y noticias**: {related_channel} + **Detalles sobre donaciones**: /donate + **LiberaPay**: {libera_pay_url} + **Dirección BTC:** {btc_donate_address} + Puedes enviarnos sus comentarios directamente por /contact + COPYRIGHT_DESCRIPTION: | + Si deseas presentar una queja por infracción de derechos de autor: + - Comienza tu mensaje con /copyright, luego agregua la nueva línea + - Pega los enlaces a los ítems sobre los que estás reclamando. Un enlace a un ítem + es aquel sobre el que estás haciendo clic en el resultado de la página de búsqueda, es decir, `/vb_FH5fWn2o_3512868_1675254_2` + - Agrega como un solo archivo (*.zip) todos los documentos que confirman que eres el propietario legal de los derechos de autor o un agente autorizado para actuar en nombre del propietario. + Tu reclamo será considerado durante 24 horas. + La decisión de satisfacer el reclamo impedirá que se descargue el ítem, pero aún se podrá buscar. + COPYRIGHT_INFRINGEMENT_ACCEPTED: | + Se ha aceptado su reclamo por infracción de derechos de autor. + COULD_NOT_FIND_ANYTHING: No pude encontrar nada :( + DESCRIPTION: Descripción + DISABLE_DISCOVERY: Si no deseas obtener un feed personalizado, puedes deshabilitar Nexus Discovery en /settings + DISCOVERY_OPTION: Nexus Discovery + DISCOVERY_PROMPT: 🦘 **Mira lo que encontré para ti:** 🦘 + DONATE: | + Gracias por visitar /donate. Agradezco incluso tu intención de abrir este mensaje. + Toda buena enseñanza debe fluir de abundantes fuentes de conocimiento. Las fuentes poco profundas no pueden emitir un resultado fluido y abundante. + Al donarnos a nosotros, estás donando a todas las personas que no tienen acceso al conocimiento humano debido a restricciones de pago o un precio excesivamente alto. + Todas las donaciones se gastarán en: + - Desarrollo del Nexus Index y hacer que todo el mundo pueda buscar tanto conocimiento como sea posible. + - Mantenimiento de servidores. Crear y entregar el índice y usar el aprendizaje automático es una tarea computacional pesada que requiere una gran cantidad de GPU, CPU y discos. + Cada centavo se utilizará para el proyecto. + Puedes contactarnos con /contact o suscribirte en {related_channel}, si deseas saber exactamente cómo se gastarán las donaciones. + **Detalles sobre donaciones** + **LiberaPay**: {libera_pay_url} + **Dirección BTC:** {btc_donate_address} + [‌]({btc_donate_qr_url}) + DOWNLOAD: Descargar + DOWNLOADED: descargado + DOWNLOADING: descargando... + DOWNLOAD_AND_SEARCH_MORE: Descarga y busca más + DOWNLOAD_CANCELED: | + Estamos recargando nuestros backends. + Por favor, intente descargar `{document}` en un minuto. + EDITION: Edición + FILE: Archivo + HELP: | + ¡Hola! ¡Soy el bot **Nexus**! + Puedo buscar libros o artículos científicos por títulos, autores o DOI. + Además, puedo enviar recomendaciones basadas en lo que estabas buscando antes. + **Simplemente escriba su solicitud en palabras sencillas.** + Ejemplo: `Cien Años de Soledad` o `https://doi.org/10.1023/A:1026654312961` + Restringir la búsqueda para el año 2019: `hemoglobina AND year:2019` + Encuentre solo a partir del año 2019: `hemoglobina AND year:[2019 TO *] + Búsqueda por autor: `authors:Jack authors:London` + Coincidencia exacta: `"Hemoglobina Fetal"` + /copyright - hacer un reclamo por infracción de derechos de autor + /donate - explica cómo apoyarnos + /help - muestra esta ayuda + /roll - obtener un libro al azar + /settings - cambia la configuración de idioma, notificaciones y feeds personalizados + ¿Qué vas a leer hoy? + HELP_FOR_GROUPS: | + ¡Hola! ¡Soy el bot **Nexus** para grupos! + Puedo buscar libros o artículos científicos por títulos, autores o DOI. + Simplemente escribe tu solicitud en palabras sencillas precedido de **/search**. + Ejemplo: `Cien Años de Soledad` o `https://doi.org/10.1023/A:1026654312961` + Restringir la búsqueda para el año 2019: `hemoglobina AND year:2019` + Encuentre solo a partir del año 2019: `hemoglobina AND year:[2019 TO *] + Búsqueda por autor: `authors:Jack authors:London` + Coincidencia exacta: `"Hemoglobina Fetal"` + /donate - explica cómo apoyarnos + /help - muestra esta ayuda + /roll - obtener un libro al azar + /settings - cambia la configuración de idioma, notificaciones y feeds personalizados + ¿Qué vas a leer hoy? + INVALID_QUERY_ERROR: | + ```Oops, No sé qué hacer con esta URL :( + ¡Intenta enviarme DOI, MD5 o simplemente el nombre de lo que estás buscando!```[‌]({too_difficult_picture_url}) + INVALID_SYNTAX_ERROR: | + ```Oops, algo anda mal con tu consulta. Probablemente sea demasiado inteligente para mí. + Tip: usa corchetes, agrega más AND/OR o quítalos todos.```[‌]({too_difficult_picture_url}) + JOURNAL: Revista + LEGACY: Hemos actualizado y los widgets antiguos dejaron de funcionar :( Por favor, vuelva a buscar. + LOOKING_AT: 'buscando en {source}...' + MAINTENANCE: | + ```¡Oops! Algo salió mal y estamos esforzándonos por revivir. + Por favor, inténtalo más tarde.```[‌]({maintenance_picture_url}) + MAINTENANCE_WO_PIC: ¡Oops! Por favor, intenta más tarde. + NAMELESS: Sin nombre + NEXUS_DISCOVERY_DESCRIPTION: | + **Nexus Discovery** es un servicio de recomendaciones. También te notifica sobre publicaciones recién llegadas que estabas buscando antes. Tu resumen personal se enviará todos los domingos. + PROMO: | + **Nexus Search - Actualizaciones y Noticias** + {related_channel} + READ_ONLY_MODE: ⚠️ Servicio reducido hasta la medianoche (UTC). /settings no funciona, busque en modo lento. + REFERENCED_BY: Referenciado por + REPLY_MESSAGE_HAS_BEEN_DELETED: El mensaje de búsqueda ha sido (re)movido. Vuelve a buscar. + SEARCHING: buscando... + SEND_YOUR_LOCATION: Envía tu ubicación (a través del botón de adjuntar de la izquierda) + SETTINGS_ROUTER_HELP: > + Los ajustes se pueden configurar de forma automática o manual. + El modo automático solicitará la ubicación para establecer la zona horaria, el idioma y la clasificación geográfica. + SETTINGS_TEMPLATE: | + **Versión del bot:** {bot_version} + **Versión de Nexus:** {nexus_version} + **Idioma:** {language} + **Zona horaria:** {tzinfo} + SETUP_AUTOMATICALLY: Configurar automáticamente + SETUP_MANUALLY: Configurar manualmente + SHORT_PROMO: ⤴️ Mantente en contacto con nosotros en {related_channel} + SOURCES_UNAVAILABLE: '`{document}` no está disponible en este momento. Por favor intenta más tarde.' + SUBSCRIBE_TO_CHANNEL: Suscríbete a {related_channel} para seguir usando este bot. + SYSTEM_MESSAGING_OPTION: Notificaciones de actualización de bot + TAGS: Tags + TANKS_BRUH: Tank ya bruuuh! + THANK_YOU_FOR_CONTACT: ¡Gracias! Si tienes alguna pregunta, es mejor que la hagas en el grupo de usuarios de {related_channel}. + TOO_LONG_QUERY_FOR_SHORTLINK: | + Su consulta es demasiado larga (el límite es de 35 a 40 caracteres). + Intente utilizar una consulta codificada con NID. Busque un elemento que desee compartir, copie el NID de una página de visualización y + luego cree un enlace corto para la consulta `NID: `. + TOO_MANY_DOWNLOADS: Demasiadas descargas activas. Por favor, espera a que finalicen. + TRANSMITTED_FROM: 'transmitido desde {source}' + UPGRADE_MAINTENANCE: | + ```Se está llevando a cabo una actualización importante, vuelve con nosotros mañana```[‌]({upgrade_maintenance_picture_url}) + UPLOADED_TO_TELEGRAM: subido a Telegram... + VIEWS_CANNOT_BE_SHARED: Haga su propia búsqueda, los comandos no se pueden compartir. + YEAR: Año ru: ABSTRACT: Аннотация ALREADY_DOWNLOADING: | diff --git a/rules/rust/Cargo.toml b/rules/rust/Cargo.toml index cf88219..4d4f1f3 100644 --- a/rules/rust/Cargo.toml +++ b/rules/rust/Cargo.toml @@ -34,6 +34,7 @@ colored = "2" combine = { version = "4", default-features = false, features = [] } config = "0.10.1" crc32fast = "1.2.0" +criterion = "0.3.3" crossbeam = "0.7.3" deadpool = "0.5.2" deadpool-postgres = "0.5.6" @@ -115,7 +116,7 @@ tls-api-stub = "0.4.0" tokio = { version = "0.2", features = ["full", "time"] } tokio-pg-mapper = "0.2.0" tokio-pg-mapper-derive = "0.2.0" -tokio-postgres = "0.7.0" +tokio-postgres = "0.5" typenum = { version = "1.12.0", features = ["force_unix_path_separator"] } urlparse = "0.7.3" uuid = { version = "0.8.1", features = ["v4", "serde"] } diff --git a/rules/rust/cargo/BUILD.bazel b/rules/rust/cargo/BUILD.bazel index d27b65d..8bd8bfd 100644 --- a/rules/rust/cargo/BUILD.bazel +++ b/rules/rust/cargo/BUILD.bazel @@ -246,6 +246,15 @@ alias( ], ) +alias( + name = "criterion", + actual = "@raze__criterion__0_3_3//:criterion", + tags = [ + "cargo-raze", + "manual", + ], +) + alias( name = "crossbeam", actual = "@raze__crossbeam__0_7_3//:crossbeam", @@ -999,7 +1008,7 @@ alias( alias( name = "tokio_postgres", - actual = "@raze__tokio_postgres__0_7_0//:tokio_postgres", + actual = "@raze__tokio_postgres__0_5_5//:tokio_postgres", tags = [ "cargo-raze", "manual", diff --git a/rules/rust/crates.bzl b/rules/rust/crates.bzl index 6f6d57e..0d99c36 100644 --- a/rules/rust/crates.bzl +++ b/rules/rust/crates.bzl @@ -461,6 +461,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.brotli-decompressor-2.3.1.bazel"), ) + maybe( + http_archive, + name = "raze__bstr__0_2_14", + url = "https://crates.io/api/v1/crates/bstr/0.2.14/download", + type = "tar.gz", + strip_prefix = "bstr-0.2.14", + build_file = Label("//rules/rust/remote:BUILD.bstr-0.2.14.bazel"), + ) + maybe( http_archive, name = "raze__bumpalo__3_4_0", @@ -506,6 +515,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.bytestring-0.1.5.bazel"), ) + maybe( + http_archive, + name = "raze__cast__0_2_3", + url = "https://crates.io/api/v1/crates/cast/0.2.3/download", + type = "tar.gz", + strip_prefix = "cast-0.2.3", + build_file = Label("//rules/rust/remote:BUILD.cast-0.2.3.bazel"), + ) + maybe( http_archive, name = "raze__cc__1_0_66", @@ -650,6 +668,24 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.crc32fast-1.2.1.bazel"), ) + maybe( + http_archive, + name = "raze__criterion__0_3_3", + url = "https://crates.io/api/v1/crates/criterion/0.3.3/download", + type = "tar.gz", + strip_prefix = "criterion-0.3.3", + build_file = Label("//rules/rust/remote:BUILD.criterion-0.3.3.bazel"), + ) + + maybe( + http_archive, + name = "raze__criterion_plot__0_4_3", + url = "https://crates.io/api/v1/crates/criterion-plot/0.4.3/download", + type = "tar.gz", + strip_prefix = "criterion-plot-0.4.3", + build_file = Label("//rules/rust/remote:BUILD.criterion-plot-0.4.3.bazel"), + ) + maybe( http_archive, name = "raze__crossbeam__0_7_3", @@ -767,6 +803,24 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.crypto-mac-0.9.1.bazel"), ) + maybe( + http_archive, + name = "raze__csv__1_1_5", + url = "https://crates.io/api/v1/crates/csv/1.1.5/download", + type = "tar.gz", + strip_prefix = "csv-1.1.5", + build_file = Label("//rules/rust/remote:BUILD.csv-1.1.5.bazel"), + ) + + maybe( + http_archive, + name = "raze__csv_core__0_1_10", + url = "https://crates.io/api/v1/crates/csv-core/0.1.10/download", + type = "tar.gz", + strip_prefix = "csv-core-0.1.10", + build_file = Label("//rules/rust/remote:BUILD.csv-core-0.1.10.bazel"), + ) + maybe( http_archive, name = "raze__deadpool__0_5_2", @@ -1181,6 +1235,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.h2-0.2.7.bazel"), ) + maybe( + http_archive, + name = "raze__half__1_6_0", + url = "https://crates.io/api/v1/crates/half/1.6.0/download", + type = "tar.gz", + strip_prefix = "half-1.6.0", + build_file = Label("//rules/rust/remote:BUILD.half-1.6.0.bazel"), + ) + maybe( http_archive, name = "raze__hashbrown__0_9_1", @@ -1352,6 +1415,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.itoa-0.4.7.bazel"), ) + maybe( + http_archive, + name = "raze__js_sys__0_3_46", + url = "https://crates.io/api/v1/crates/js-sys/0.3.46/download", + type = "tar.gz", + strip_prefix = "js-sys-0.3.46", + build_file = Label("//rules/rust/remote:BUILD.js-sys-0.3.46.bazel"), + ) + maybe( http_archive, name = "raze__kernel32_sys__0_2_2", @@ -1766,6 +1838,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.once_cell-1.5.2.bazel"), ) + maybe( + http_archive, + name = "raze__oorandom__11_1_3", + url = "https://crates.io/api/v1/crates/oorandom/11.1.3/download", + type = "tar.gz", + strip_prefix = "oorandom-11.1.3", + build_file = Label("//rules/rust/remote:BUILD.oorandom-11.1.3.bazel"), + ) + maybe( http_archive, name = "raze__opaque_debug__0_3_0", @@ -1910,6 +1991,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.pin-utils-0.1.0.bazel"), ) + maybe( + http_archive, + name = "raze__plotters__0_2_15", + url = "https://crates.io/api/v1/crates/plotters/0.2.15/download", + type = "tar.gz", + strip_prefix = "plotters-0.2.15", + build_file = Label("//rules/rust/remote:BUILD.plotters-0.2.15.bazel"), + ) + maybe( http_archive, name = "raze__postgres__0_17_5", @@ -2234,6 +2324,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.regex-1.4.2.bazel"), ) + maybe( + http_archive, + name = "raze__regex_automata__0_1_9", + url = "https://crates.io/api/v1/crates/regex-automata/0.1.9/download", + type = "tar.gz", + strip_prefix = "regex-automata-0.1.9", + build_file = Label("//rules/rust/remote:BUILD.regex-automata-0.1.9.bazel"), + ) + maybe( http_archive, name = "raze__regex_syntax__0_4_2", @@ -2360,6 +2459,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.safemem-0.3.3.bazel"), ) + maybe( + http_archive, + name = "raze__same_file__1_0_6", + url = "https://crates.io/api/v1/crates/same-file/1.0.6/download", + type = "tar.gz", + strip_prefix = "same-file-1.0.6", + build_file = Label("//rules/rust/remote:BUILD.same-file-1.0.6.bazel"), + ) + maybe( http_archive, name = "raze__scheduled_thread_pool__0_2_5", @@ -2432,6 +2540,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.serde-value-0.6.0.bazel"), ) + maybe( + http_archive, + name = "raze__serde_cbor__0_11_1", + url = "https://crates.io/api/v1/crates/serde_cbor/0.11.1/download", + type = "tar.gz", + strip_prefix = "serde_cbor-0.11.1", + build_file = Label("//rules/rust/remote:BUILD.serde_cbor-0.11.1.bazel"), + ) + maybe( http_archive, name = "raze__serde_derive__1_0_118", @@ -2936,6 +3053,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.time-macros-impl-0.1.1.bazel"), ) + maybe( + http_archive, + name = "raze__tinytemplate__1_2_0", + url = "https://crates.io/api/v1/crates/tinytemplate/1.2.0/download", + type = "tar.gz", + strip_prefix = "tinytemplate-1.2.0", + build_file = Label("//rules/rust/remote:BUILD.tinytemplate-1.2.0.bazel"), + ) + maybe( http_archive, name = "raze__tinyvec__1_1_0", @@ -3287,6 +3413,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.wait-timeout-0.2.0.bazel"), ) + maybe( + http_archive, + name = "raze__walkdir__2_3_1", + url = "https://crates.io/api/v1/crates/walkdir/2.3.1/download", + type = "tar.gz", + strip_prefix = "walkdir-2.3.1", + build_file = Label("//rules/rust/remote:BUILD.walkdir-2.3.1.bazel"), + ) + maybe( http_archive, name = "raze__wasi__0_10_0_wasi_snapshot_preview1", @@ -3350,6 +3485,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.wasm-bindgen-shared-0.2.69.bazel"), ) + maybe( + http_archive, + name = "raze__web_sys__0_3_46", + url = "https://crates.io/api/v1/crates/web-sys/0.3.46/download", + type = "tar.gz", + strip_prefix = "web-sys-0.3.46", + build_file = Label("//rules/rust/remote:BUILD.web-sys-0.3.46.bazel"), + ) + maybe( http_archive, name = "raze__widestring__0_4_3", @@ -3395,6 +3539,15 @@ def raze_fetch_remote_crates(): build_file = Label("//rules/rust/remote:BUILD.winapi-i686-pc-windows-gnu-0.4.0.bazel"), ) + maybe( + http_archive, + name = "raze__winapi_util__0_1_5", + url = "https://crates.io/api/v1/crates/winapi-util/0.1.5/download", + type = "tar.gz", + strip_prefix = "winapi-util-0.1.5", + build_file = Label("//rules/rust/remote:BUILD.winapi-util-0.1.5.bazel"), + ) + maybe( http_archive, name = "raze__winapi_x86_64_pc_windows_gnu__0_4_0", diff --git a/rules/rust/remote/BUILD.bstr-0.2.14.bazel b/rules/rust/remote/BUILD.bstr-0.2.14.bazel new file mode 100644 index 0000000..5c68d7a --- /dev/null +++ b/rules/rust/remote/BUILD.bstr-0.2.14.bazel @@ -0,0 +1,81 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +# Unsupported target "graphemes" with type "example" omitted + +# Unsupported target "graphemes-std" with type "example" omitted + +# Unsupported target "lines" with type "example" omitted + +# Unsupported target "lines-std" with type "example" omitted + +# Unsupported target "uppercase" with type "example" omitted + +# Unsupported target "uppercase-std" with type "example" omitted + +# Unsupported target "words" with type "example" omitted + +# Unsupported target "words-std" with type "example" omitted + +rust_library( + name = "bstr", + srcs = glob(["**/*.rs"]), + crate_features = [ + "default", + "lazy_static", + "regex-automata", + "serde", + "serde1", + "serde1-nostd", + "std", + "unicode", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2015", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.2.14", + # buildifier: leave-alone + deps = [ + "@raze__lazy_static__1_4_0//:lazy_static", + "@raze__memchr__2_3_4//:memchr", + "@raze__regex_automata__0_1_9//:regex_automata", + "@raze__serde__1_0_118//:serde", + ], +) diff --git a/rules/rust/remote/BUILD.cast-0.2.3.bazel b/rules/rust/remote/BUILD.cast-0.2.3.bazel new file mode 100644 index 0000000..1f12816 --- /dev/null +++ b/rules/rust/remote/BUILD.cast-0.2.3.bazel @@ -0,0 +1,87 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets +# buildifier: disable=load-on-top +load( + "@io_bazel_rules_rust//cargo:cargo_build_script.bzl", + "cargo_build_script", +) + +cargo_build_script( + name = "cast_build_script", + srcs = glob(["**/*.rs"]), + build_script_env = { + }, + crate_features = [ + "default", + "std", + ], + crate_root = "build.rs", + data = glob(["**"]), + edition = "2015", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.2.3", + visibility = ["//visibility:private"], + deps = [ + "@raze__rustc_version__0_2_3//:rustc_version", + ], +) + +rust_library( + name = "cast", + srcs = glob(["**/*.rs"]), + crate_features = [ + "default", + "std", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2015", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.2.3", + # buildifier: leave-alone + deps = [ + ":cast_build_script", + ], +) diff --git a/rules/rust/remote/BUILD.criterion-0.3.3.bazel b/rules/rust/remote/BUILD.criterion-0.3.3.bazel new file mode 100644 index 0000000..e510bea --- /dev/null +++ b/rules/rust/remote/BUILD.criterion-0.3.3.bazel @@ -0,0 +1,78 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # Apache-2.0 from expression "Apache-2.0 OR MIT" +]) + +# Generated Targets + +# Unsupported target "bench_main" with type "bench" omitted + +rust_library( + name = "criterion", + srcs = glob(["**/*.rs"]), + crate_features = [ + "default", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + proc_macro_deps = [ + "@raze__serde_derive__1_0_118//:serde_derive", + ], + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.3.3", + # buildifier: leave-alone + deps = [ + "@raze__atty__0_2_14//:atty", + "@raze__cast__0_2_3//:cast", + "@raze__clap__2_33_3//:clap", + "@raze__criterion_plot__0_4_3//:criterion_plot", + "@raze__csv__1_1_5//:csv", + "@raze__itertools__0_9_0//:itertools", + "@raze__lazy_static__1_4_0//:lazy_static", + "@raze__num_traits__0_2_14//:num_traits", + "@raze__oorandom__11_1_3//:oorandom", + "@raze__plotters__0_2_15//:plotters", + "@raze__rayon__1_5_0//:rayon", + "@raze__regex__1_4_2//:regex", + "@raze__serde__1_0_118//:serde", + "@raze__serde_cbor__0_11_1//:serde_cbor", + "@raze__serde_json__1_0_61//:serde_json", + "@raze__tinytemplate__1_2_0//:tinytemplate", + "@raze__walkdir__2_3_1//:walkdir", + ], +) + +# Unsupported target "criterion_tests" with type "test" omitted diff --git a/rules/rust/remote/BUILD.criterion-plot-0.4.3.bazel b/rules/rust/remote/BUILD.criterion-plot-0.4.3.bazel new file mode 100644 index 0000000..2204a86 --- /dev/null +++ b/rules/rust/remote/BUILD.criterion-plot-0.4.3.bazel @@ -0,0 +1,55 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +rust_library( + name = "criterion_plot", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.4.3", + # buildifier: leave-alone + deps = [ + "@raze__cast__0_2_3//:cast", + "@raze__itertools__0_9_0//:itertools", + ], +) diff --git a/rules/rust/remote/BUILD.csv-1.1.5.bazel b/rules/rust/remote/BUILD.csv-1.1.5.bazel new file mode 100644 index 0000000..3437438 --- /dev/null +++ b/rules/rust/remote/BUILD.csv-1.1.5.bazel @@ -0,0 +1,134 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "unencumbered", # Unlicense from expression "Unlicense OR MIT" +]) + +# Generated Targets + +# Unsupported target "bench" with type "bench" omitted + +# Unsupported target "cookbook-read-basic" with type "example" omitted + +# Unsupported target "cookbook-read-colon" with type "example" omitted + +# Unsupported target "cookbook-read-no-headers" with type "example" omitted + +# Unsupported target "cookbook-read-serde" with type "example" omitted + +# Unsupported target "cookbook-write-basic" with type "example" omitted + +# Unsupported target "cookbook-write-serde" with type "example" omitted + +# Unsupported target "tutorial-error-01" with type "example" omitted + +# Unsupported target "tutorial-error-02" with type "example" omitted + +# Unsupported target "tutorial-error-03" with type "example" omitted + +# Unsupported target "tutorial-error-04" with type "example" omitted + +# Unsupported target "tutorial-perf-alloc-01" with type "example" omitted + +# Unsupported target "tutorial-perf-alloc-02" with type "example" omitted + +# Unsupported target "tutorial-perf-alloc-03" with type "example" omitted + +# Unsupported target "tutorial-perf-core-01" with type "example" omitted + +# Unsupported target "tutorial-perf-serde-01" with type "example" omitted + +# Unsupported target "tutorial-perf-serde-02" with type "example" omitted + +# Unsupported target "tutorial-perf-serde-03" with type "example" omitted + +# Unsupported target "tutorial-pipeline-pop-01" with type "example" omitted + +# Unsupported target "tutorial-pipeline-search-01" with type "example" omitted + +# Unsupported target "tutorial-pipeline-search-02" with type "example" omitted + +# Unsupported target "tutorial-read-01" with type "example" omitted + +# Unsupported target "tutorial-read-delimiter-01" with type "example" omitted + +# Unsupported target "tutorial-read-headers-01" with type "example" omitted + +# Unsupported target "tutorial-read-headers-02" with type "example" omitted + +# Unsupported target "tutorial-read-serde-01" with type "example" omitted + +# Unsupported target "tutorial-read-serde-02" with type "example" omitted + +# Unsupported target "tutorial-read-serde-03" with type "example" omitted + +# Unsupported target "tutorial-read-serde-04" with type "example" omitted + +# Unsupported target "tutorial-read-serde-invalid-01" with type "example" omitted + +# Unsupported target "tutorial-read-serde-invalid-02" with type "example" omitted + +# Unsupported target "tutorial-setup-01" with type "example" omitted + +# Unsupported target "tutorial-write-01" with type "example" omitted + +# Unsupported target "tutorial-write-02" with type "example" omitted + +# Unsupported target "tutorial-write-delimiter-01" with type "example" omitted + +# Unsupported target "tutorial-write-serde-01" with type "example" omitted + +# Unsupported target "tutorial-write-serde-02" with type "example" omitted + +rust_library( + name = "csv", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "1.1.5", + # buildifier: leave-alone + deps = [ + "@raze__bstr__0_2_14//:bstr", + "@raze__csv_core__0_1_10//:csv_core", + "@raze__itoa__0_4_7//:itoa", + "@raze__ryu__1_0_5//:ryu", + "@raze__serde__1_0_118//:serde", + ], +) + +# Unsupported target "tests" with type "test" omitted diff --git a/rules/rust/remote/BUILD.csv-core-0.1.10.bazel b/rules/rust/remote/BUILD.csv-core-0.1.10.bazel new file mode 100644 index 0000000..c344245 --- /dev/null +++ b/rules/rust/remote/BUILD.csv-core-0.1.10.bazel @@ -0,0 +1,57 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "unencumbered", # Unlicense from expression "Unlicense OR MIT" +]) + +# Generated Targets + +# Unsupported target "bench" with type "bench" omitted + +rust_library( + name = "csv_core", + srcs = glob(["**/*.rs"]), + crate_features = [ + "default", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.1.10", + # buildifier: leave-alone + deps = [ + "@raze__memchr__2_3_4//:memchr", + ], +) diff --git a/rules/rust/remote/BUILD.half-1.6.0.bazel b/rules/rust/remote/BUILD.half-1.6.0.bazel new file mode 100644 index 0000000..de0553e --- /dev/null +++ b/rules/rust/remote/BUILD.half-1.6.0.bazel @@ -0,0 +1,57 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +# Unsupported target "convert" with type "bench" omitted + +rust_library( + name = "half", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "1.6.0", + # buildifier: leave-alone + deps = [ + ], +) + +# Unsupported target "version-numbers" with type "test" omitted diff --git a/rules/rust/remote/BUILD.js-sys-0.3.46.bazel b/rules/rust/remote/BUILD.js-sys-0.3.46.bazel new file mode 100644 index 0000000..dbcf218 --- /dev/null +++ b/rules/rust/remote/BUILD.js-sys-0.3.46.bazel @@ -0,0 +1,58 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +rust_library( + name = "js_sys", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.3.46", + # buildifier: leave-alone + deps = [ + "@raze__wasm_bindgen__0_2_69//:wasm_bindgen", + ], +) + +# Unsupported target "headless" with type "test" omitted + +# Unsupported target "wasm" with type "test" omitted diff --git a/rules/rust/remote/BUILD.oorandom-11.1.3.bazel b/rules/rust/remote/BUILD.oorandom-11.1.3.bazel new file mode 100644 index 0000000..b0853c4 --- /dev/null +++ b/rules/rust/remote/BUILD.oorandom-11.1.3.bazel @@ -0,0 +1,53 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT" +]) + +# Generated Targets + +rust_library( + name = "oorandom", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "11.1.3", + # buildifier: leave-alone + deps = [ + ], +) diff --git a/rules/rust/remote/BUILD.plotters-0.2.15.bazel b/rules/rust/remote/BUILD.plotters-0.2.15.bazel new file mode 100644 index 0000000..dcff614 --- /dev/null +++ b/rules/rust/remote/BUILD.plotters-0.2.15.bazel @@ -0,0 +1,105 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT" +]) + +# Generated Targets + +# Unsupported target "benchmark" with type "bench" omitted + +# Unsupported target "animation" with type "example" omitted + +# Unsupported target "area-chart" with type "example" omitted + +# Unsupported target "blit-bitmap" with type "example" omitted + +# Unsupported target "boxplot" with type "example" omitted + +# Unsupported target "chart" with type "example" omitted + +# Unsupported target "console" with type "example" omitted + +# Unsupported target "errorbar" with type "example" omitted + +# Unsupported target "histogram" with type "example" omitted + +# Unsupported target "mandelbrot" with type "example" omitted + +# Unsupported target "matshow" with type "example" omitted + +# Unsupported target "normal-dist" with type "example" omitted + +# Unsupported target "normal-dist2" with type "example" omitted + +# Unsupported target "relative_size" with type "example" omitted + +# Unsupported target "sierpinski" with type "example" omitted + +# Unsupported target "slc-temp" with type "example" omitted + +# Unsupported target "snowflake" with type "example" omitted + +# Unsupported target "stock" with type "example" omitted + +# Unsupported target "two-scales" with type "example" omitted + +rust_library( + name = "plotters", + srcs = glob(["**/*.rs"]), + aliases = { + }, + crate_features = [ + "area_series", + "line_series", + "svg", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.2.15", + # buildifier: leave-alone + deps = [ + "@raze__num_traits__0_2_14//:num_traits", + ] + selects.with_or({ + # cfg(not(target_arch = "wasm32")) + ( + "@io_bazel_rules_rust//rust/platform:x86_64-apple-darwin", + "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu", + ): [ + ], + "//conditions:default": [], + }), +) diff --git a/rules/rust/remote/BUILD.regex-automata-0.1.9.bazel b/rules/rust/remote/BUILD.regex-automata-0.1.9.bazel new file mode 100644 index 0000000..05fa7b4 --- /dev/null +++ b/rules/rust/remote/BUILD.regex-automata-0.1.9.bazel @@ -0,0 +1,56 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "unencumbered", # Unlicense from expression "Unlicense OR MIT" +]) + +# Generated Targets + +rust_library( + name = "regex_automata", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2015", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.1.9", + # buildifier: leave-alone + deps = [ + "@raze__byteorder__1_3_4//:byteorder", + ], +) + +# Unsupported target "default" with type "test" omitted diff --git a/rules/rust/remote/BUILD.same-file-1.0.6.bazel b/rules/rust/remote/BUILD.same-file-1.0.6.bazel new file mode 100644 index 0000000..1695f15 --- /dev/null +++ b/rules/rust/remote/BUILD.same-file-1.0.6.bazel @@ -0,0 +1,57 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "unencumbered", # Unlicense from expression "Unlicense OR MIT" +]) + +# Generated Targets + +# Unsupported target "is_same_file" with type "example" omitted + +# Unsupported target "is_stderr" with type "example" omitted + +rust_library( + name = "same_file", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "1.0.6", + # buildifier: leave-alone + deps = [ + ], +) diff --git a/rules/rust/remote/BUILD.serde_cbor-0.11.1.bazel b/rules/rust/remote/BUILD.serde_cbor-0.11.1.bazel new file mode 100644 index 0000000..d063e27 --- /dev/null +++ b/rules/rust/remote/BUILD.serde_cbor-0.11.1.bazel @@ -0,0 +1,77 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +# Unsupported target "readme" with type "example" omitted + +# Unsupported target "tags" with type "example" omitted + +rust_library( + name = "serde_cbor", + srcs = glob(["**/*.rs"]), + crate_features = [ + "default", + "std", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.11.1", + # buildifier: leave-alone + deps = [ + "@raze__half__1_6_0//:half", + "@raze__serde__1_0_118//:serde", + ], +) + +# Unsupported target "bennofs" with type "test" omitted + +# Unsupported target "canonical" with type "test" omitted + +# Unsupported target "de" with type "test" omitted + +# Unsupported target "enum" with type "test" omitted + +# Unsupported target "ser" with type "test" omitted + +# Unsupported target "std_types" with type "test" omitted + +# Unsupported target "tags" with type "test" omitted + +# Unsupported target "value" with type "test" omitted diff --git a/rules/rust/remote/BUILD.tinytemplate-1.2.0.bazel b/rules/rust/remote/BUILD.tinytemplate-1.2.0.bazel new file mode 100644 index 0000000..7392757 --- /dev/null +++ b/rules/rust/remote/BUILD.tinytemplate-1.2.0.bazel @@ -0,0 +1,57 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # Apache-2.0 from expression "Apache-2.0 OR MIT" +]) + +# Generated Targets + +# Unsupported target "benchmarks" with type "bench" omitted + +rust_library( + name = "tinytemplate", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2015", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "1.2.0", + # buildifier: leave-alone + deps = [ + "@raze__serde__1_0_118//:serde", + "@raze__serde_json__1_0_61//:serde_json", + ], +) diff --git a/rules/rust/remote/BUILD.walkdir-2.3.1.bazel b/rules/rust/remote/BUILD.walkdir-2.3.1.bazel new file mode 100644 index 0000000..f3e4d84 --- /dev/null +++ b/rules/rust/remote/BUILD.walkdir-2.3.1.bazel @@ -0,0 +1,54 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "unencumbered", # Unlicense from expression "Unlicense OR MIT" +]) + +# Generated Targets + +rust_library( + name = "walkdir", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "2.3.1", + # buildifier: leave-alone + deps = [ + "@raze__same_file__1_0_6//:same_file", + ], +) diff --git a/rules/rust/remote/BUILD.web-sys-0.3.46.bazel b/rules/rust/remote/BUILD.web-sys-0.3.46.bazel new file mode 100644 index 0000000..bb8693e --- /dev/null +++ b/rules/rust/remote/BUILD.web-sys-0.3.46.bazel @@ -0,0 +1,67 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "notice", # MIT from expression "MIT OR Apache-2.0" +]) + +# Generated Targets + +rust_library( + name = "web_sys", + srcs = glob(["**/*.rs"]), + crate_features = [ + "CanvasRenderingContext2d", + "Document", + "DomRect", + "DomRectReadOnly", + "Element", + "EventTarget", + "HtmlCanvasElement", + "HtmlElement", + "Node", + "Window", + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.3.46", + # buildifier: leave-alone + deps = [ + "@raze__js_sys__0_3_46//:js_sys", + "@raze__wasm_bindgen__0_2_69//:wasm_bindgen", + ], +) + +# Unsupported target "wasm" with type "test" omitted diff --git a/rules/rust/remote/BUILD.winapi-util-0.1.5.bazel b/rules/rust/remote/BUILD.winapi-util-0.1.5.bazel new file mode 100644 index 0000000..d528e3e --- /dev/null +++ b/rules/rust/remote/BUILD.winapi-util-0.1.5.bazel @@ -0,0 +1,53 @@ +""" +@generated +cargo-raze crate build file. + +DO NOT EDIT! Replaced on runs of cargo-raze +""" + +# buildifier: disable=load +load( + "@io_bazel_rules_rust//rust:rust.bzl", + "rust_binary", + "rust_library", + "rust_test", +) + +# buildifier: disable=load +load("@bazel_skylib//lib:selects.bzl", "selects") + +package(default_visibility = [ + # Public for visibility by "@raze__crate__version//" targets. + # + # Prefer access through "//rules/rust", which limits external + # visibility to explicit Cargo.toml dependencies. + "//visibility:public", +]) + +licenses([ + "unencumbered", # Unlicense from expression "Unlicense OR MIT" +]) + +# Generated Targets + +rust_library( + name = "winapi_util", + srcs = glob(["**/*.rs"]), + crate_features = [ + ], + crate_root = "src/lib.rs", + crate_type = "lib", + data = [], + edition = "2018", + rustc_flags = [ + "--cap-lints=allow", + ], + tags = [ + "cargo-raze", + "manual", + ], + version = "0.1.5", + # buildifier: leave-alone + deps = [ + ], +)