- Send Pylon to golden

-  Move fancy naming to Hub
-  Adopt translations
-  Fix long authors list in bot
-  Support CJK in scitech
-  Fixed pylon bug with decoding improper unicode ch...

GitOrigin-RevId: 74f73c44f749a71cb65dd5ddd3416f32a83d329f

images/production/BUILD.bazel

@@ -9,6 +9,7 @@ download_pkgs(
     image_tar = "//images:base-image.tar",
     packages = [
         "bash",
+        "ca-certificates",
         "libev4",
         "libgomp1",
         "libgoogle-perftools-dev",

nexus/README.md

@@ -11,7 +11,7 @@
 - ✅ [`models`](models) - shared Protobuf models
 - ✅ [`nlptools`](nlptools) - text routines
 - ✅ [`pipe`](pipe) - processing pipeline based on Kafka
-- 🛑 `pylon` - smart proxy for downloading files from the Internet/IPFS
+- ✅ [`pylon`](pylon) - smart client for downloading files from the Internet/IPFS
 - ✅ [`summa`](summa) - scripts for setting Summa
 - ✅ [`translations`](translations) - text translations used in `bot` and `hub`
 - 🛑 `views` - shared views for [`models`](models)

nexus/pylon/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
+load("@pip_modules_external//:requirements.bzl", "requirement")
+
+py_library(
+    name = "pylon",
+    srcs = glob(["**/*.py"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("aiodns"),
+        requirement("aiohttp"),
+        requirement("aiohttp_socks"),
+        requirement("brotli"),
+        requirement("cchardet"),
+        requirement("orjson"),
+        requirement("pdfminer.six"),
+        requirement("python-socks"),
+        requirement("tenacity"),
+        requirement("aiokit"),
+        "//library/logging",
+        "//nexus/pylon/proto:pylon_proto_py",
+    ],
+)
+
+py_binary(
+    name = "cli",
+    srcs = ["cli.py"],
+    main = "cli.py",
+    srcs_version = "PY3",
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("fire"),
+        ":pylon",
+    ],
+)

nexus/pylon/README.md

@@ -0,0 +1,21 @@
+# Nexus Pylon
+
+`Pylon` is a downloader for scientific publications.
+- Look articles by DOI, MD5 or IPFS hashes
+- Validates downloaded items
+- Streams data by chunks
+- GRPC-ready
+
+## Nexus Pylon CLI
+
+Casual download
+```bash 
+bazel run -c opt cli -- doi 10.1056/NEJMoa2033700 --output article.pdf
+```
+
+Download with proxy
+```bash 
+bazel run -c opt cli -- md5 278C3A72B7B04717361501B8642857DF \
+  --output file.pdf \
+  --proxy socks5://127.0.0.1:9050
+```

nexus/pylon/cli.py

@@ -0,0 +1,81 @@
+import os
+from typing import Optional
+
+import fire
+from aiokit.utils import sync_fu
+from nexus.pylon.client import (
+    DownloadError,
+    PylonClient,
+)
+from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb
+
+
+def resolve_path(filepath):
+    if os.path.isabs(filepath):
+        return filepath
+    cwd = os.environ.get('BUILD_WORKING_DIRECTORY', os.getcwd())
+    filepath = os.path.join(cwd, filepath)
+    return filepath
+
+
+async def fetch(
+    iter,
+    output: str,
+):
+    collected = bytes()
+    try:
+        last_len = 0
+        last_source = ''
+        async for resp in iter:
+            if resp.HasField('status'):
+                if resp.status == FileResponsePb.Status.BEGIN_TRANSMISSION:
+                    print(f'Started transmission from {resp.source}...', end='\r')
+                    last_len = 0
+                    last_source = resp.source
+                    collected = bytes()
+            elif resp.HasField('chunk'):
+                if len(collected) - last_len > 1024 * 100:
+                    print(f'Loaded {len(collected)} bytes from {resp.source}', end='\r')
+                    last_len = len(collected)
+                    last_source = resp.source
+                collected += resp.chunk.content
+        with open(resolve_path(output), 'wb') as f:
+            print(f'Completed! Loaded {len(collected)} bytes from {last_source}')
+            f.write(collected)
+    except DownloadError:
+        print('File not found')
+
+
+async def fetch_by_md5(
+    md5: str,
+    output: str,
+    proxy: Optional[str] = None,
+    resolve_proxy: Optional[str] = None,
+):
+    if proxy and not resolve_proxy:
+        resolve_proxy = proxy
+    p = PylonClient(proxy=proxy, resolve_proxy=resolve_proxy)
+    return await fetch(iter=p.by_md5(md5=md5), output=output)
+
+
+async def fetch_by_doi(
+    doi: str,
+    output: str,
+    proxy: Optional[str] = None,
+    resolve_proxy: Optional[str] = None,
+):
+    if proxy and not resolve_proxy:
+        resolve_proxy = proxy
+    p = PylonClient(proxy=proxy, resolve_proxy=resolve_proxy)
+    return await fetch(iter=p.by_doi(doi=doi), output=output)
+
+
+def main():
+    fire.Fire({
+        'doi': sync_fu(fetch_by_doi),
+        'md5': sync_fu(fetch_by_md5),
+    })
+
+
+if __name__ == '__main__':
+    main()

nexus/pylon/client.py

@@ -0,0 +1,98 @@
+import asyncio
+from typing import (
+    AsyncIterable,
+    Iterable,
+    Optional,
+)
+
+import aiohttp
+import aiohttp.client_exceptions
+from aiohttp_socks import (
+    ProxyConnectionError,
+    ProxyError,
+)
+from aiokit import AioThing
+from library.logging import error_log
+from nexus.pylon.exceptions import (
+    DownloadError,
+    NotFoundError,
+)
+from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb
+from nexus.pylon.sources import (
+    BaseSource,
+    LibgenDoiSource,
+    LibgenMd5Source,
+    LibraryLolSource,
+    SciHubSeSource,
+    SciHubTwSource,
+)
+from nexus.pylon.sources.specific import get_specific_sources_for_doi
+from python_socks import ProxyTimeoutError
+
+
+class PylonClient(AioThing):
+    def __init__(self, proxy: Optional[str] = None, resolve_proxy: Optional[str] = None):
+        super().__init__()
+        self.proxy = proxy
+        self.resolve_proxy = resolve_proxy
+
+    async def by_doi(
+        self,
+        doi: str,
+        md5: Optional[str] = None,
+    ) -> AsyncIterable[FileResponsePb]:
+        sources = []
+        sources.extend(get_specific_sources_for_doi(doi, proxy=self.proxy, resolve_proxy=self.resolve_proxy))
+        sources.extend([
+            SciHubSeSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
+            SciHubTwSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
+            LibgenDoiSource(doi=doi, md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
+        ])
+        sources = filter(lambda x: x.is_enabled, sources)
+        async for resp in self.download(sources=sources):
+            yield resp
+
+    async def by_md5(
+        self,
+        md5: str,
+    ) -> AsyncIterable[FileResponsePb]:
+        sources = filter(lambda x: x.is_enabled, [
+            LibraryLolSource(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
+            LibgenMd5Source(md5=md5, proxy=self.proxy, resolve_proxy=self.resolve_proxy),
+        ])
+        async for resp in self.download(sources=sources):
+            yield resp
+
+    async def download_source(self, source) -> AsyncIterable[FileResponsePb]:
+        try:
+            yield FileResponsePb(status=FileResponsePb.Status.RESOLVING, source=source.base_url)
+            async for prepared_request in source.resolve():
+                async for resp in source.execute_prepared_request(prepared_request=prepared_request):
+                    yield resp
+                return
+            raise DownloadError(error='not_found', source=str(source))
+        except (
+            aiohttp.client_exceptions.ClientConnectionError,
+            aiohttp.client_exceptions.ClientPayloadError,
+            aiohttp.client_exceptions.ClientResponseError,
+            aiohttp.client_exceptions.TooManyRedirects,
+            asyncio.TimeoutError,
+            ProxyConnectionError,
+            ProxyTimeoutError,
+            ProxyError,
+        ) as e:
+            raise DownloadError(nested_error=str(e), nested_error_cls=e.__class__.__name__)
+
+    async def download(self, sources: Iterable[BaseSource]) -> AsyncIterable[FileResponsePb]:
+        for source in sources:
+            try:
+                await source.start()
+                async for resp in self.download_source(source):
+                    yield resp
+                return
+            except DownloadError as e:
+                error_log(e)
+                continue
+            finally:
+                await source.stop()
+        raise NotFoundError()

nexus/pylon/exceptions.py

@@ -0,0 +1,39 @@
+import logging
+
+from izihawa_utils.exceptions import BaseError
+
+
+class DownloadError(BaseError):
+    level = logging.WARNING
+    code = 'download_error'
+
+
+class IncorrectMD5Error(DownloadError):
+    code = 'incorrect_md5_error'
+
+
+class UnresolvableSourceError(DownloadError):
+    level = logging.WARNING
+    code = 'unresolvable_source_error'
+
+
+class UnavailableSourcesError(DownloadError):
+    code = 'no_available_sources_error'
+
+
+class NotFoundError(DownloadError):
+    level = logging.WARNING
+    code = 'not_found_error'
+
+
+class RegexNotFoundError(DownloadError):
+    level = logging.WARNING
+    code = 'regex_not_found_error'
+
+
+class BadResponseError(DownloadError):
+    code = 'bad_response_error'
+
+
+class InvalidDocumentForDownload(DownloadError):
+    code = 'invalid_document_for_download'

nexus/pylon/pdftools/__init__.py

@@ -0,0 +1,9 @@
+from pdfminer.high_level import extract_text
+
+
+def is_pdf(path_to_file):
+    try:
+        extract_text(path_to_file, maxpages=1)
+        return True
+    except Exception:
+        return False

nexus/pylon/proto/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library")
+load("@rules_proto//proto:defs.bzl", "proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_library(
+    name = "pylon_proto",
+    srcs = glob([
+        "*.proto",
+    ]),
+    deps = [
+        "@com_google_protobuf//:wrappers_proto",
+    ],
+)
+
+py_proto_library(
+    name = "pylon_proto_py",
+    deps = [":pylon_proto"],
+)

nexus/pylon/proto/file.proto

@@ -0,0 +1,18 @@
+syntax = "proto3";
+package nexus.pylon.proto;
+
+message Chunk {
+  bytes content = 1;
+}
+
+message FileResponse {
+  enum Status {
+    RESOLVING = 0;
+    BEGIN_TRANSMISSION = 1;
+  }
+  oneof response {
+    Chunk chunk = 1;
+    Status status = 2;
+  }
+  string source = 3;
+}

nexus/pylon/proto/file_pb2.py

@@ -0,0 +1,165 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: nexus/pylon/proto/file.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+  name='nexus/pylon/proto/file.proto',
+  package='nexus.pylon.proto',
+  syntax='proto3',
+  serialized_options=None,
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x1cnexus/pylon/proto/file.proto\x12\x11nexus.pylon.proto\"\x18\n\x05\x43hunk\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\"\xc0\x01\n\x0c\x46ileResponse\x12)\n\x05\x63hunk\x18\x01 \x01(\x0b\x32\x18.nexus.pylon.proto.ChunkH\x00\x12\x38\n\x06status\x18\x02 \x01(\x0e\x32&.nexus.pylon.proto.FileResponse.StatusH\x00\x12\x0e\n\x06source\x18\x03 \x01(\t\"/\n\x06Status\x12\r\n\tRESOLVING\x10\x00\x12\x16\n\x12\x42\x45GIN_TRANSMISSION\x10\x01\x42\n\n\x08responseb\x06proto3'
+)
+
+
+
+_FILERESPONSE_STATUS = _descriptor.EnumDescriptor(
+  name='Status',
+  full_name='nexus.pylon.proto.FileResponse.Status',
+  filename=None,
+  file=DESCRIPTOR,
+  create_key=_descriptor._internal_create_key,
+  values=[
+    _descriptor.EnumValueDescriptor(
+      name='RESOLVING', index=0, number=0,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+    _descriptor.EnumValueDescriptor(
+      name='BEGIN_TRANSMISSION', index=1, number=1,
+      serialized_options=None,
+      type=None,
+      create_key=_descriptor._internal_create_key),
+  ],
+  containing_type=None,
+  serialized_options=None,
+  serialized_start=211,
+  serialized_end=258,
+)
+_sym_db.RegisterEnumDescriptor(_FILERESPONSE_STATUS)
+
+
+_CHUNK = _descriptor.Descriptor(
+  name='Chunk',
+  full_name='nexus.pylon.proto.Chunk',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='content', full_name='nexus.pylon.proto.Chunk.content', index=0,
+      number=1, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"",
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=51,
+  serialized_end=75,
+)
+
+
+_FILERESPONSE = _descriptor.Descriptor(
+  name='FileResponse',
+  full_name='nexus.pylon.proto.FileResponse',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='chunk', full_name='nexus.pylon.proto.FileResponse.chunk', index=0,
+      number=1, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='status', full_name='nexus.pylon.proto.FileResponse.status', index=1,
+      number=2, type=14, cpp_type=8, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='source', full_name='nexus.pylon.proto.FileResponse.source', index=2,
+      number=3, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=b"".decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+    _FILERESPONSE_STATUS,
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+    _descriptor.OneofDescriptor(
+      name='response', full_name='nexus.pylon.proto.FileResponse.response',
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
+  ],
+  serialized_start=78,
+  serialized_end=270,
+)
+
+_FILERESPONSE.fields_by_name['chunk'].message_type = _CHUNK
+_FILERESPONSE.fields_by_name['status'].enum_type = _FILERESPONSE_STATUS
+_FILERESPONSE_STATUS.containing_type = _FILERESPONSE
+_FILERESPONSE.oneofs_by_name['response'].fields.append(
+  _FILERESPONSE.fields_by_name['chunk'])
+_FILERESPONSE.fields_by_name['chunk'].containing_oneof = _FILERESPONSE.oneofs_by_name['response']
+_FILERESPONSE.oneofs_by_name['response'].fields.append(
+  _FILERESPONSE.fields_by_name['status'])
+_FILERESPONSE.fields_by_name['status'].containing_oneof = _FILERESPONSE.oneofs_by_name['response']
+DESCRIPTOR.message_types_by_name['Chunk'] = _CHUNK
+DESCRIPTOR.message_types_by_name['FileResponse'] = _FILERESPONSE
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+Chunk = _reflection.GeneratedProtocolMessageType('Chunk', (_message.Message,), {
+  'DESCRIPTOR' : _CHUNK,
+  '__module__' : 'nexus.pylon.proto.file_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.pylon.proto.Chunk)
+  })
+_sym_db.RegisterMessage(Chunk)
+
+FileResponse = _reflection.GeneratedProtocolMessageType('FileResponse', (_message.Message,), {
+  'DESCRIPTOR' : _FILERESPONSE,
+  '__module__' : 'nexus.pylon.proto.file_pb2'
+  # @@protoc_insertion_point(class_scope:nexus.pylon.proto.FileResponse)
+  })
+_sym_db.RegisterMessage(FileResponse)
+
+
+# @@protoc_insertion_point(module_scope)

nexus/pylon/sources/__init__.py

@@ -0,0 +1,19 @@
+from .base import BaseSource
+from .libgen_doi import LibgenDoiSource
+from .libgen_md5 import LibgenMd5Source
+from .libgen_new import (
+    LibraryLolSource,
+)
+from .scihub import (
+    SciHubSeSource,
+    SciHubTwSource,
+)
+
+__all__ = [
+    'BaseSource',
+    'LibgenDoiSource',
+    'LibgenMd5Source',
+    'LibraryLolSource',
+    'SciHubSeSource',
+    'SciHubTwSource',
+]

nexus/pylon/sources/base.py

@@ -0,0 +1,202 @@
+import hashlib
+import random
+from typing import (
+    AsyncIterable,
+    Optional,
+)
+
+import aiohttp
+import aiohttp.client_exceptions
+from aiohttp_socks import (
+    ProxyConnector,
+    ProxyError,
+)
+from aiokit import AioThing
+from nexus.pylon.exceptions import (
+    BadResponseError,
+    IncorrectMD5Error,
+    NotFoundError,
+)
+from nexus.pylon.pdftools import is_pdf
+from nexus.pylon.proto.file_pb2 import Chunk as ChunkPb
+from nexus.pylon.proto.file_pb2 import FileResponse as FileResponsePb
+from python_socks import ProxyTimeoutError
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+)
+
+DEFAULT_USER_AGENT = 'PylonBot/1.0 (Linux x86_64) PylonBot/1.0.0'
+
+
+class PreparedRequest:
+    def __init__(
+        self,
+        method: str,
+        url: str,
+        headers: dict = None,
+        params: dict = None,
+        cookies: dict = None,
+        ssl: bool = True,
+    ):
+        self.method = method
+        self.url = url
+        self.headers = {
+            'User-Agent': DEFAULT_USER_AGENT,
+        }
+        if headers:
+            self.headers.update(headers)
+        self.params = params
+        self.cookies = cookies
+        self.ssl = ssl
+
+    def __repr__(self):
+        return f'{self.method} {self.url} {self.headers} {self.params}'
+
+    def __str__(self):
+        return repr(self)
+
+
+class BaseValidator:
+    def update(self, chunk: bytes):
+        pass
+
+    def validate(self):
+        pass
+
+
+class Md5Validator(BaseValidator):
+    def __init__(self, md5: str):
+        self.md5 = md5
+        self.v = hashlib.md5()
+
+    def update(self, chunk: bytes):
+        self.v.update(chunk)
+
+    def validate(self):
+        digest = self.v.hexdigest()
+        if self.md5.lower() != self.v.hexdigest().lower():
+            raise IncorrectMD5Error(requested_md5=self.md5, downloaded_md5=digest)
+
+
+class DoiValidator(BaseValidator):
+    def __init__(self, doi: str):
+        self.doi = doi
+        self.file = bytes()
+
+    def update(self, chunk):
+        self.file += chunk
+
+    def validate(self):
+        return is_pdf(self.file)
+
+
+class BaseSource(AioThing):
+    allowed_content_type = None
+    base_url = None
+    is_enabled = True
+    resolve_timeout = None
+    ssl = True
+    timeout = None
+    use_proxy = None
+
+    def __init__(self, proxy: str = None, resolve_proxy: str = None):
+        super().__init__()
+        self.proxy = proxy
+        self.resolve_proxy = resolve_proxy
+
+    def get_proxy(self):
+        if self.proxy and self.use_proxy is not False:
+            return ProxyConnector.from_url(self.proxy, verify_ssl=self.ssl)
+        return aiohttp.TCPConnector(verify_ssl=self.ssl)
+
+    def get_resolve_proxy(self):
+        if self.resolve_proxy and self.use_proxy is not False:
+            return ProxyConnector.from_url(self.resolve_proxy, verify_ssl=self.ssl)
+        return aiohttp.TCPConnector(verify_ssl=self.ssl)
+
+    def get_session(self):
+        return aiohttp.ClientSession(connector=self.get_proxy())
+
+    def get_resolve_session(self):
+        return aiohttp.ClientSession(connector=self.get_resolve_proxy())
+
+    def resolve(self) -> AsyncIterable[PreparedRequest]:
+        raise NotImplementedError("`resolve` for BaseSource is not implemented")
+
+    def get_validator(self):
+        return BaseValidator()
+
+    @retry(
+        reraise=True,
+        stop=stop_after_attempt(3),
+        retry=retry_if_exception_type((ProxyError, aiohttp.client_exceptions.ClientPayloadError, ProxyTimeoutError)),
+    )
+    async def execute_prepared_request(self, prepared_request: PreparedRequest):
+        async with self.get_session() as session:
+            async with session.request(
+                method=prepared_request.method,
+                url=prepared_request.url,
+                timeout=self.timeout,
+                headers=prepared_request.headers,
+                cookies=prepared_request.cookies,
+                params=prepared_request.params,
+                ssl=prepared_request.ssl,
+            ) as resp:
+                if resp.status == 404:
+                    raise NotFoundError(url=prepared_request.url)
+                elif (
+                    resp.status != 200
+                    or (
+                        self.allowed_content_type
+                        and resp.headers.get('Content-Type', '').lower() not in self.allowed_content_type
+                    )
+                ):
+                    raise BadResponseError(
+                        request_headers=prepared_request.headers,
+                        url=prepared_request.url,
+                        status=resp.status,
+                        headers=str(resp.headers),
+                    )
+                file_validator = self.get_validator()
+                # Randomness is required due to annoying bug of when separators
+                # (\r\n) are splitted to different chunks
+                # https://github.com/aio-libs/aiohttp/issues/4677
+                yield FileResponsePb(status=FileResponsePb.Status.BEGIN_TRANSMISSION, source=prepared_request.url)
+                async for content in resp.content.iter_chunked(1024 * 100 + random.randint(-1024, 1024)):
+                    file_validator.update(content)
+                    yield FileResponsePb(chunk=ChunkPb(content=content), source=prepared_request.url)
+                file_validator.validate()
+
+
+class Md5Source(BaseSource):
+    def __init__(
+        self,
+        md5: str,
+        proxy: Optional[str] = None,
+        resolve_proxy: Optional[str] = None,
+    ):
+        super().__init__(proxy=proxy, resolve_proxy=resolve_proxy)
+        self.md5 = md5
+
+    def get_validator(self):
+        return Md5Validator(self.md5)
+
+
+class DoiSource(BaseSource):
+    def __init__(
+        self,
+        doi: str,
+        md5: Optional[str] = None,
+        proxy: Optional[str] = None,
+        resolve_proxy: Optional[str] = None,
+    ):
+        super().__init__(proxy=proxy, resolve_proxy=resolve_proxy)
+        self.doi = doi
+        self.md5 = md5
+
+    def get_validator(self):
+        if self.md5:
+            return Md5Validator(self.md5)
+        return DoiValidator(self.doi)

nexus/pylon/sources/libgen_doi.py

@@ -0,0 +1,34 @@
+import re
+from typing import AsyncIterable
+
+from library.logging import error_log
+from nexus.pylon.exceptions import RegexNotFoundError
+
+from .base import (
+    DoiSource,
+    PreparedRequest,
+)
+
+
+class LibgenDoiSource(DoiSource):
+    base_url = 'http://libgen.gs'
+    resolve_timeout = 10
+
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/scimag/ads.php?doi={self.doi}'
+            async with session.get(
+                url,
+                timeout=self.resolve_timeout
+            ) as resp:
+                downloaded_page_bytes = await resp.read()
+                downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace')
+            match = re.search(
+                'https?://.*/scimag/get\\.php\\?doi=.*&key=[A-Za-z0-9]+',
+                downloaded_page,
+                re.IGNORECASE,
+            )
+            if match:
+                yield PreparedRequest(method='get', url=match.group())
+            else:
+                error_log(RegexNotFoundError(url=url))

nexus/pylon/sources/libgen_md5.py

@@ -0,0 +1,37 @@
+import re
+from typing import AsyncIterable
+
+from .base import (
+    Md5Source,
+    PreparedRequest,
+)
+
+
+class LibgenMd5Source(Md5Source):
+    base_url = 'http://libgen.gs'
+    resolve_timeout = 10
+
+    async def resolve_lg(self, session, url):
+        async with session.get(
+            url,
+            timeout=self.resolve_timeout
+        ) as resp:
+            downloaded_page_fiction = await resp.text()
+        match = re.search(
+            'https?://.*/get\\.php\\?md5=.*&key=[A-Za-z0-9]+',
+            downloaded_page_fiction,
+            re.IGNORECASE,
+        )
+        if match:
+            return PreparedRequest(method='get', url=match.group())
+
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/ads.php?md5={self.md5}'
+            result = await self.resolve_lg(session, url)
+            if result:
+                yield result
+            url = f'{self.base_url}/foreignfiction/ads.php?md5={self.md5}'
+            result = await self.resolve_lg(session, url)
+            if result:
+                yield result

nexus/pylon/sources/libgen_new.py

@@ -0,0 +1,52 @@
+import re
+from typing import AsyncIterable
+
+from library.logging import error_log
+from nexus.pylon.exceptions import RegexNotFoundError
+
+from .base import (
+    Md5Source,
+    PreparedRequest,
+)
+
+
+class LibgenNewSource(Md5Source):
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/main/{self.md5.upper()}'
+            async with session.get(
+                url,
+                timeout=self.resolve_timeout
+            ) as resp:
+                downloaded_page = await resp.text()
+            match_ipfs = re.search(
+                'https://ipfs.io/ipfs/[A-Za-z0-9]+',
+                downloaded_page,
+                re.IGNORECASE,
+            )
+            if match_ipfs:
+                yield PreparedRequest(method='get', url=match_ipfs.group(), ssl=self.ssl)
+            match_cf = re.search(
+                'https://cloudflare-ipfs.com/ipfs/[A-Za-z0-9]+',
+                downloaded_page,
+                re.IGNORECASE,
+            )
+            if match_cf:
+                yield PreparedRequest(method='get', url=match_cf.group(), ssl=self.ssl)
+            match_infura = re.search(
+                'https://ipfs.infura.io/ipfs/[A-Za-z0-9]+',
+                downloaded_page,
+                re.IGNORECASE,
+            )
+            if match_infura:
+                yield PreparedRequest(method='get', url=match_infura.group(), ssl=self.ssl)
+
+            if not match_cf or not match_infura or not match_ipfs:
+                error_log(RegexNotFoundError(url=url))
+
+
+class LibraryLolSource(LibgenNewSource):
+    base_url = 'http://library.lol'
+    resolve_timeout = 10
+    ssl = False
+    timeout = 20

nexus/pylon/sources/scihub.py

@@ -0,0 +1,49 @@
+import re
+from typing import AsyncIterable
+
+from library.logging import error_log
+from nexus.pylon.exceptions import RegexNotFoundError
+
+from .base import (
+    DoiSource,
+    PreparedRequest,
+)
+
+
+class SciHubSource(DoiSource):
+    allowed_content_type = {
+        'application/octet-stream',
+        'application/pdf',
+        'application/pdf;charset=utf-8',
+    }
+    base_url = None
+    ssl = False
+
+    async def resolve(self, timeout=None) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/{self.doi}'
+            async with session.get(
+                url,
+                timeout=timeout or self.timeout
+            ) as resp:
+                # Sometimes sci-hub returns file
+                if resp.headers.get('Content-Type') == 'application/pdf':
+                    yield PreparedRequest(method='get', url=url)
+                downloaded_page_bytes = await resp.read()
+                downloaded_page = downloaded_page_bytes.decode('utf-8', 'backslashreplace')
+            match = re.search('(?:https?:)?//.*\\?download=true', downloaded_page, re.IGNORECASE)
+            if match:
+                url = match.group()
+                if url.startswith('//'):
+                    url = 'http:' + url
+                yield PreparedRequest(method='get', url=url)
+            else:
+                error_log(RegexNotFoundError(url=url))
+
+
+class SciHubSeSource(SciHubSource):
+    base_url = 'https://sci-hub.se'
+
+
+class SciHubTwSource(SciHubSource):
+    base_url = 'https://sci-hub.tw'

nexus/pylon/sources/specific/__init__.py

@@ -0,0 +1,20 @@
+from typing import Iterable
+
+from ..base import DoiSource
+from .biorxiv import BiorxivSource
+from .lancet import LancetSource
+from .nejm import NejmSource
+from .research_square import ResearchSquareSource
+
+paper_sources = {
+    '10.1016': [LancetSource],
+    '10.1056': [NejmSource],
+    '10.1101': [BiorxivSource],
+    '10.21203': [ResearchSquareSource],
+}
+
+
+def get_specific_sources_for_doi(doi: str, **kwargs) -> Iterable[DoiSource]:
+    source_clses = paper_sources.get(doi.split('/')[0], [])
+    source_clses = list(map(lambda cls: cls(doi, **kwargs), source_clses))
+    return source_clses

nexus/pylon/sources/specific/biorxiv.py

@@ -0,0 +1,19 @@
+from typing import AsyncIterable
+
+from nexus.pylon.sources.base import (
+    DoiSource,
+    PreparedRequest,
+)
+
+
+class BiorxivSource(DoiSource):
+    base_url = 'https://dx.doi.org'
+
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/{self.doi}'
+            async with session.get(
+                url,
+                timeout=self.resolve_timeout
+            ) as resp:
+                yield PreparedRequest(method='get', url=str(resp.url) + '.full.pdf')

nexus/pylon/sources/specific/lancet.py

@@ -0,0 +1,24 @@
+from typing import AsyncIterable
+
+from nexus.pylon.sources.base import (
+    DoiSource,
+    PreparedRequest,
+)
+
+
+class LancetSource(DoiSource):
+    base_url = 'https://www.thelancet.com'
+    resolve_timeout = 10
+    use_proxy = False
+
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            splitted_doi = self.doi.split("/", maxsplit=1)
+            if len(splitted_doi) < 2:
+                return
+            url = f'{self.base_url}/action/showPdf?pii={splitted_doi[1].upper()}'
+            async with session.get(
+                url,
+                timeout=self.resolve_timeout
+            ) as resp:
+                yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))

nexus/pylon/sources/specific/nejm.py

@@ -0,0 +1,21 @@
+from typing import AsyncIterable
+
+from nexus.pylon.sources.base import (
+    DoiSource,
+    PreparedRequest,
+)
+
+
+class NejmSource(DoiSource):
+    base_url = 'https://www.nejm.org'
+    resolve_timeout = 10
+    use_proxy = False
+
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/doi/pdf/{self.doi}'
+            async with session.get(
+                url,
+                timeout=self.resolve_timeout
+            ) as resp:
+                yield PreparedRequest(method='get', cookies=resp.cookies, url=str(resp.url))

nexus/pylon/sources/specific/research_square.py

@@ -0,0 +1,30 @@
+import re
+from typing import AsyncIterable
+
+from nexus.pylon.exceptions import RegexNotFoundError
+
+from nexus.pylon.sources.base import (
+    DoiSource,
+    PreparedRequest,
+)
+
+
+class ResearchSquareSource(DoiSource):
+    base_url = 'https://dx.doi.org'
+
+    async def resolve(self) -> AsyncIterable[PreparedRequest]:
+        async with self.get_resolve_session() as session:
+            url = f'{self.base_url}/{self.doi}'
+            async with session.get(
+                url,
+                timeout=self.resolve_timeout
+            ) as resp:
+                download_page = await resp.text()
+            match = re.search(
+                r'\"(https://www\.researchsquare\.com/article/[^\"]+\.pdf)\"',
+                download_page,
+                re.IGNORECASE,
+            )
+            if not match:
+                raise RegexNotFoundError(url=url)
+            yield PreparedRequest(method='get', url=match.group(1))

nexus/summa/configs/config.yaml

@@ -7,7 +7,7 @@ http:
   workers: 4
 log_path: /var/log/summa/{{ ENV_TYPE }}
 search_engine:
-  data_path: /summa
+  data_path: /summa/20210108
   default_page_size: 5
   timeout_secs: 5
   writer_memory_mb: 1024

nexus/summa/schema/scitech.yaml

@@ -17,7 +17,7 @@ schema:
     options:
       indexing:
         record: position
-        tokenizer: default
+        tokenizer: summa
       stored: true
   - name: cu
     type: text
@@ -34,7 +34,7 @@ schema:
     options:
       indexing:
         record: position
-        tokenizer: default
+        tokenizer: summa
       stored: true
   - name: doi
     type: text
@@ -120,21 +120,21 @@ schema:
     options:
       indexing:
         record: position
-        tokenizer: default
+        tokenizer: summa
       stored: true
   - name: tags
     type: text
     options:
       indexing:
         record: position
-        tokenizer: default
+        tokenizer: summa
       stored: true
   - name: title
     type: text
     options:
       indexing:
         record: position
-        tokenizer: default
+        tokenizer: summa
       stored: true
   - name: updated_at
     type: i64

nexus/translations/translations.yaml

@@ -24,7 +24,7 @@ en:
 
     You can directly feedback us by /contact <your message here>
   COPYRIGHT_DESCRIPTION: |
-    If you with to make a copyright infringement complaint:
+    If you wish to make a copyright infringement complaint:
 
     - Start your message with /copyright, then add the new line
     - Paste view links of items you are claiming about. View link is an item
@@ -160,6 +160,145 @@ en:
   UPLOADED_TO_TELEGRAM: uploaded to Telegram...
   VIEWS_CANNOT_BE_SHARED: Do your own search, commands cannot be shared.
   YEAR: Year
+es:
+  ABSTRACT: Resumen
+  ALREADY_DOWNLOADING: |
+    Encontré este libro en Internet.
+    Por favor, ten paciencia, te lo enviaré cuando esté listo.
+  AUTHORS: Autores
+  BANNED: |
+    Has sido baneado hasta la(s) **{datetime} UTC**.
+    Razón: **{reason}**
+  BANNED_FOR_SECONDS: |
+    Has sido baneado por **{seconds} segundo(s)**
+    Razón: **{reason}**
+  BAN_MESSAGE_TOO_MANY_REQUESTS: Demasiadas peticiones
+  CLOSE: Cerrar
+  CONTACT: |
+    **Actualizaciones y noticias**: {related_channel}
+    **Detalles sobre donaciones**: /donate
+    **LiberaPay**: {libera_pay_url}
+    **Dirección BTC:** {btc_donate_address}
+    Puedes enviarnos sus comentarios directamente por /contact <tu mensaje aquí>
+  COPYRIGHT_DESCRIPTION: |
+    Si deseas presentar una queja por infracción de derechos de autor:
+    - Comienza tu mensaje con /copyright, luego agregua la nueva línea
+    - Pega los enlaces a los ítems sobre los que estás reclamando. Un enlace a un ítem
+    es aquel sobre el que estás haciendo clic en el resultado de la página de búsqueda, es decir, `/vb_FH5fWn2o_3512868_1675254_2`
+    - Agrega como un solo archivo (*.zip) todos los documentos que confirman que eres el propietario legal de los derechos de autor o un agente autorizado para actuar en nombre del propietario.
+    Tu reclamo será considerado durante 24 horas.
+    La decisión de satisfacer el reclamo impedirá que se descargue el ítem, pero aún se podrá buscar.
+  COPYRIGHT_INFRINGEMENT_ACCEPTED: |
+    Se ha aceptado su reclamo por infracción de derechos de autor.
+  COULD_NOT_FIND_ANYTHING: No pude encontrar nada :(
+  DESCRIPTION: Descripción
+  DISABLE_DISCOVERY: Si no deseas obtener un feed personalizado, puedes deshabilitar Nexus Discovery en /settings
+  DISCOVERY_OPTION: Nexus Discovery
+  DISCOVERY_PROMPT: 🦘 **Mira lo que encontré para ti:** 🦘
+  DONATE: |
+    Gracias por visitar /donate. Agradezco incluso tu intención de abrir este mensaje.
+    Toda buena enseñanza debe fluir de abundantes fuentes de conocimiento. Las fuentes poco profundas no pueden emitir un resultado fluido y abundante.
+    Al donarnos a nosotros, estás donando a todas las personas que no tienen acceso al conocimiento humano debido a restricciones de pago o un precio excesivamente alto.
+    Todas las donaciones se gastarán en:
+    - Desarrollo del Nexus Index y hacer que todo el mundo pueda buscar tanto conocimiento como sea posible.
+    - Mantenimiento de servidores. Crear y entregar el índice y usar el aprendizaje automático es una tarea computacional pesada que requiere una gran cantidad de GPU, CPU y discos.
+    Cada centavo se utilizará para el proyecto.
+    Puedes contactarnos con /contact o suscribirte en {related_channel}, si deseas saber exactamente cómo se gastarán las donaciones.
+    **Detalles sobre donaciones**
+    **LiberaPay**: {libera_pay_url}
+    **Dirección BTC:** {btc_donate_address}
+    [‌]({btc_donate_qr_url})
+  DOWNLOAD: Descargar
+  DOWNLOADED: descargado
+  DOWNLOADING: descargando...
+  DOWNLOAD_AND_SEARCH_MORE: Descarga y busca más
+  DOWNLOAD_CANCELED: |
+    Estamos recargando nuestros backends.
+    Por favor, intente descargar `{document}` en un minuto.
+  EDITION: Edición
+  FILE: Archivo
+  HELP: |
+    ¡Hola! ¡Soy el bot **Nexus**!
+    Puedo buscar libros o artículos científicos por títulos, autores o DOI.
+    Además, puedo enviar recomendaciones basadas en lo que estabas buscando antes.
+    **Simplemente escriba su solicitud en palabras sencillas.**
+    Ejemplo: `Cien Años de Soledad` o `https://doi.org/10.1023/A:1026654312961`
+    Restringir la búsqueda para el año 2019: `hemoglobina AND year:2019`
+    Encuentre solo a partir del año 2019: `hemoglobina AND year:[2019 TO *]
+    Búsqueda por autor: `authors:Jack authors:London`
+    Coincidencia exacta: `"Hemoglobina Fetal"`
+    /copyright - hacer un reclamo por infracción de derechos de autor
+    /donate - explica cómo apoyarnos
+    /help - muestra esta ayuda
+    /roll - obtener un libro al azar
+    /settings - cambia la configuración de idioma, notificaciones y feeds personalizados
+    ¿Qué vas a leer hoy?
+  HELP_FOR_GROUPS: |
+    ¡Hola! ¡Soy el bot **Nexus** para grupos!
+    Puedo buscar libros o artículos científicos por títulos, autores o DOI.
+    Simplemente escribe tu solicitud en palabras sencillas precedido de **/search**.
+    Ejemplo: `Cien Años de Soledad` o `https://doi.org/10.1023/A:1026654312961`
+    Restringir la búsqueda para el año 2019: `hemoglobina AND year:2019`
+    Encuentre solo a partir del año 2019: `hemoglobina AND year:[2019 TO *]
+    Búsqueda por autor: `authors:Jack authors:London`
+    Coincidencia exacta: `"Hemoglobina Fetal"`
+    /donate - explica cómo apoyarnos
+    /help - muestra esta ayuda
+    /roll - obtener un libro al azar
+    /settings - cambia la configuración de idioma, notificaciones y feeds personalizados
+    ¿Qué vas a leer hoy?
+  INVALID_QUERY_ERROR: |
+    ```Oops, No sé qué hacer con esta URL :(
+    ¡Intenta enviarme DOI, MD5 o simplemente el nombre de lo que estás buscando!```[‌]({too_difficult_picture_url})
+  INVALID_SYNTAX_ERROR: |
+    ```Oops, algo anda mal con tu consulta. Probablemente sea demasiado inteligente para mí.
+    Tip: usa corchetes, agrega más AND/OR o quítalos todos.```[‌]({too_difficult_picture_url})
+  JOURNAL: Revista
+  LEGACY: Hemos actualizado y los widgets antiguos dejaron de funcionar :( Por favor, vuelva a buscar.
+  LOOKING_AT: 'buscando en {source}...'
+  MAINTENANCE: |
+    ```¡Oops! Algo salió mal y estamos esforzándonos por revivir.
+    Por favor, inténtalo más tarde.```[‌]({maintenance_picture_url})
+  MAINTENANCE_WO_PIC: ¡Oops! Por favor, intenta más tarde.
+  NAMELESS: Sin nombre
+  NEXUS_DISCOVERY_DESCRIPTION: |
+    **Nexus Discovery** es un servicio de recomendaciones. También te notifica sobre publicaciones recién llegadas que estabas buscando antes. Tu resumen personal se enviará todos los domingos.
+  PROMO: |
+    **Nexus Search - Actualizaciones y Noticias**
+    {related_channel}
+  READ_ONLY_MODE: ⚠️ Servicio reducido hasta la medianoche (UTC). /settings no funciona, busque en modo lento.
+  REFERENCED_BY: Referenciado por
+  REPLY_MESSAGE_HAS_BEEN_DELETED: El mensaje de búsqueda ha sido (re)movido. Vuelve a buscar.
+  SEARCHING: buscando...
+  SEND_YOUR_LOCATION: Envía tu ubicación (a través del botón de adjuntar de la izquierda)
+  SETTINGS_ROUTER_HELP: >
+    Los ajustes se pueden configurar de forma automática o manual.
+    El modo automático solicitará la ubicación para establecer la zona horaria, el idioma y la clasificación geográfica.
+  SETTINGS_TEMPLATE: |
+    **Versión del bot:** {bot_version}
+    **Versión de Nexus:** {nexus_version}
+    **Idioma:** {language}
+    **Zona horaria:** {tzinfo}
+  SETUP_AUTOMATICALLY: Configurar automáticamente
+  SETUP_MANUALLY: Configurar manualmente
+  SHORT_PROMO: ⤴️ Mantente en contacto con nosotros en {related_channel}
+  SOURCES_UNAVAILABLE: '`{document}` no está disponible en este momento. Por favor intenta más tarde.'
+  SUBSCRIBE_TO_CHANNEL: Suscríbete a {related_channel} para seguir usando este bot.
+  SYSTEM_MESSAGING_OPTION: Notificaciones de actualización de bot
+  TAGS: Tags
+  TANKS_BRUH: Tank ya bruuuh!
+  THANK_YOU_FOR_CONTACT: ¡Gracias! Si tienes alguna pregunta, es mejor que la hagas en el grupo de usuarios de {related_channel}.
+  TOO_LONG_QUERY_FOR_SHORTLINK: |
+    Su consulta es demasiado larga (el límite es de 35 a 40 caracteres).
+    Intente utilizar una consulta codificada con NID. Busque un elemento que desee compartir, copie el NID de una página de visualización y
+    luego cree un enlace corto para la consulta `NID: <NID>`.
+  TOO_MANY_DOWNLOADS: Demasiadas descargas activas. Por favor, espera a que finalicen.
+  TRANSMITTED_FROM: 'transmitido desde {source}'
+  UPGRADE_MAINTENANCE: |
+    ```Se está llevando a cabo una actualización importante, vuelve con nosotros mañana```[‌]({upgrade_maintenance_picture_url})
+  UPLOADED_TO_TELEGRAM: subido a Telegram...
+  VIEWS_CANNOT_BE_SHARED: Haga su propia búsqueda, los comandos no se pueden compartir.
+  YEAR: Año
 ru:
   ABSTRACT: Аннотация
   ALREADY_DOWNLOADING: |

rules/rust/Cargo.toml

@@ -34,6 +34,7 @@ colored = "2"
 combine = { version = "4", default-features = false, features = [] }
 config = "0.10.1"
 crc32fast = "1.2.0"
+criterion = "0.3.3"
 crossbeam = "0.7.3"
 deadpool = "0.5.2"
 deadpool-postgres = "0.5.6"
@@ -115,7 +116,7 @@ tls-api-stub = "0.4.0"
 tokio = { version = "0.2", features = ["full", "time"] }
 tokio-pg-mapper = "0.2.0"
 tokio-pg-mapper-derive = "0.2.0"
-tokio-postgres = "0.7.0"
+tokio-postgres = "0.5"
 typenum = { version = "1.12.0", features = ["force_unix_path_separator"] }
 urlparse = "0.7.3"
 uuid = { version = "0.8.1", features = ["v4", "serde"] }

rules/rust/cargo/BUILD.bazel

@@ -246,6 +246,15 @@ alias(
     ],
 )
 
+alias(
+    name = "criterion",
+    actual = "@raze__criterion__0_3_3//:criterion",
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+)
+
 alias(
     name = "crossbeam",
     actual = "@raze__crossbeam__0_7_3//:crossbeam",
@@ -999,7 +1008,7 @@ alias(
 
 alias(
     name = "tokio_postgres",
-    actual = "@raze__tokio_postgres__0_7_0//:tokio_postgres",
+    actual = "@raze__tokio_postgres__0_5_5//:tokio_postgres",
     tags = [
         "cargo-raze",
         "manual",

rules/rust/crates.bzl

@@ -461,6 +461,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.brotli-decompressor-2.3.1.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__bstr__0_2_14",
+        url = "https://crates.io/api/v1/crates/bstr/0.2.14/download",
+        type = "tar.gz",
+        strip_prefix = "bstr-0.2.14",
+        build_file = Label("//rules/rust/remote:BUILD.bstr-0.2.14.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__bumpalo__3_4_0",
@@ -506,6 +515,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.bytestring-0.1.5.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__cast__0_2_3",
+        url = "https://crates.io/api/v1/crates/cast/0.2.3/download",
+        type = "tar.gz",
+        strip_prefix = "cast-0.2.3",
+        build_file = Label("//rules/rust/remote:BUILD.cast-0.2.3.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__cc__1_0_66",
@@ -650,6 +668,24 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.crc32fast-1.2.1.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__criterion__0_3_3",
+        url = "https://crates.io/api/v1/crates/criterion/0.3.3/download",
+        type = "tar.gz",
+        strip_prefix = "criterion-0.3.3",
+        build_file = Label("//rules/rust/remote:BUILD.criterion-0.3.3.bazel"),
+    )
+
+    maybe(
+        http_archive,
+        name = "raze__criterion_plot__0_4_3",
+        url = "https://crates.io/api/v1/crates/criterion-plot/0.4.3/download",
+        type = "tar.gz",
+        strip_prefix = "criterion-plot-0.4.3",
+        build_file = Label("//rules/rust/remote:BUILD.criterion-plot-0.4.3.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__crossbeam__0_7_3",
@@ -767,6 +803,24 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.crypto-mac-0.9.1.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__csv__1_1_5",
+        url = "https://crates.io/api/v1/crates/csv/1.1.5/download",
+        type = "tar.gz",
+        strip_prefix = "csv-1.1.5",
+        build_file = Label("//rules/rust/remote:BUILD.csv-1.1.5.bazel"),
+    )
+
+    maybe(
+        http_archive,
+        name = "raze__csv_core__0_1_10",
+        url = "https://crates.io/api/v1/crates/csv-core/0.1.10/download",
+        type = "tar.gz",
+        strip_prefix = "csv-core-0.1.10",
+        build_file = Label("//rules/rust/remote:BUILD.csv-core-0.1.10.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__deadpool__0_5_2",
@@ -1181,6 +1235,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.h2-0.2.7.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__half__1_6_0",
+        url = "https://crates.io/api/v1/crates/half/1.6.0/download",
+        type = "tar.gz",
+        strip_prefix = "half-1.6.0",
+        build_file = Label("//rules/rust/remote:BUILD.half-1.6.0.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__hashbrown__0_9_1",
@@ -1352,6 +1415,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.itoa-0.4.7.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__js_sys__0_3_46",
+        url = "https://crates.io/api/v1/crates/js-sys/0.3.46/download",
+        type = "tar.gz",
+        strip_prefix = "js-sys-0.3.46",
+        build_file = Label("//rules/rust/remote:BUILD.js-sys-0.3.46.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__kernel32_sys__0_2_2",
@@ -1766,6 +1838,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.once_cell-1.5.2.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__oorandom__11_1_3",
+        url = "https://crates.io/api/v1/crates/oorandom/11.1.3/download",
+        type = "tar.gz",
+        strip_prefix = "oorandom-11.1.3",
+        build_file = Label("//rules/rust/remote:BUILD.oorandom-11.1.3.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__opaque_debug__0_3_0",
@@ -1910,6 +1991,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.pin-utils-0.1.0.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__plotters__0_2_15",
+        url = "https://crates.io/api/v1/crates/plotters/0.2.15/download",
+        type = "tar.gz",
+        strip_prefix = "plotters-0.2.15",
+        build_file = Label("//rules/rust/remote:BUILD.plotters-0.2.15.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__postgres__0_17_5",
@@ -2234,6 +2324,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.regex-1.4.2.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__regex_automata__0_1_9",
+        url = "https://crates.io/api/v1/crates/regex-automata/0.1.9/download",
+        type = "tar.gz",
+        strip_prefix = "regex-automata-0.1.9",
+        build_file = Label("//rules/rust/remote:BUILD.regex-automata-0.1.9.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__regex_syntax__0_4_2",
@@ -2360,6 +2459,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.safemem-0.3.3.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__same_file__1_0_6",
+        url = "https://crates.io/api/v1/crates/same-file/1.0.6/download",
+        type = "tar.gz",
+        strip_prefix = "same-file-1.0.6",
+        build_file = Label("//rules/rust/remote:BUILD.same-file-1.0.6.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__scheduled_thread_pool__0_2_5",
@@ -2432,6 +2540,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.serde-value-0.6.0.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__serde_cbor__0_11_1",
+        url = "https://crates.io/api/v1/crates/serde_cbor/0.11.1/download",
+        type = "tar.gz",
+        strip_prefix = "serde_cbor-0.11.1",
+        build_file = Label("//rules/rust/remote:BUILD.serde_cbor-0.11.1.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__serde_derive__1_0_118",
@@ -2936,6 +3053,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.time-macros-impl-0.1.1.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__tinytemplate__1_2_0",
+        url = "https://crates.io/api/v1/crates/tinytemplate/1.2.0/download",
+        type = "tar.gz",
+        strip_prefix = "tinytemplate-1.2.0",
+        build_file = Label("//rules/rust/remote:BUILD.tinytemplate-1.2.0.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__tinyvec__1_1_0",
@@ -3287,6 +3413,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.wait-timeout-0.2.0.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__walkdir__2_3_1",
+        url = "https://crates.io/api/v1/crates/walkdir/2.3.1/download",
+        type = "tar.gz",
+        strip_prefix = "walkdir-2.3.1",
+        build_file = Label("//rules/rust/remote:BUILD.walkdir-2.3.1.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__wasi__0_10_0_wasi_snapshot_preview1",
@@ -3350,6 +3485,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.wasm-bindgen-shared-0.2.69.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__web_sys__0_3_46",
+        url = "https://crates.io/api/v1/crates/web-sys/0.3.46/download",
+        type = "tar.gz",
+        strip_prefix = "web-sys-0.3.46",
+        build_file = Label("//rules/rust/remote:BUILD.web-sys-0.3.46.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__widestring__0_4_3",
@@ -3395,6 +3539,15 @@ def raze_fetch_remote_crates():
         build_file = Label("//rules/rust/remote:BUILD.winapi-i686-pc-windows-gnu-0.4.0.bazel"),
     )
 
+    maybe(
+        http_archive,
+        name = "raze__winapi_util__0_1_5",
+        url = "https://crates.io/api/v1/crates/winapi-util/0.1.5/download",
+        type = "tar.gz",
+        strip_prefix = "winapi-util-0.1.5",
+        build_file = Label("//rules/rust/remote:BUILD.winapi-util-0.1.5.bazel"),
+    )
+
     maybe(
         http_archive,
         name = "raze__winapi_x86_64_pc_windows_gnu__0_4_0",

rules/rust/remote/BUILD.bstr-0.2.14.bazel

@@ -0,0 +1,81 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+# Unsupported target "graphemes" with type "example" omitted
+
+# Unsupported target "graphemes-std" with type "example" omitted
+
+# Unsupported target "lines" with type "example" omitted
+
+# Unsupported target "lines-std" with type "example" omitted
+
+# Unsupported target "uppercase" with type "example" omitted
+
+# Unsupported target "uppercase-std" with type "example" omitted
+
+# Unsupported target "words" with type "example" omitted
+
+# Unsupported target "words-std" with type "example" omitted
+
+rust_library(
+    name = "bstr",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+        "default",
+        "lazy_static",
+        "regex-automata",
+        "serde",
+        "serde1",
+        "serde1-nostd",
+        "std",
+        "unicode",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.2.14",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__lazy_static__1_4_0//:lazy_static",
+        "@raze__memchr__2_3_4//:memchr",
+        "@raze__regex_automata__0_1_9//:regex_automata",
+        "@raze__serde__1_0_118//:serde",
+    ],
+)

rules/rust/remote/BUILD.cast-0.2.3.bazel

@@ -0,0 +1,87 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+# buildifier: disable=load-on-top
+load(
+    "@io_bazel_rules_rust//cargo:cargo_build_script.bzl",
+    "cargo_build_script",
+)
+
+cargo_build_script(
+    name = "cast_build_script",
+    srcs = glob(["**/*.rs"]),
+    build_script_env = {
+    },
+    crate_features = [
+        "default",
+        "std",
+    ],
+    crate_root = "build.rs",
+    data = glob(["**"]),
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.2.3",
+    visibility = ["//visibility:private"],
+    deps = [
+        "@raze__rustc_version__0_2_3//:rustc_version",
+    ],
+)
+
+rust_library(
+    name = "cast",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+        "default",
+        "std",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.2.3",
+    # buildifier: leave-alone
+    deps = [
+        ":cast_build_script",
+    ],
+)

rules/rust/remote/BUILD.criterion-0.3.3.bazel

@@ -0,0 +1,78 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # Apache-2.0 from expression "Apache-2.0 OR MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "bench_main" with type "bench" omitted
+
+rust_library(
+    name = "criterion",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+        "default",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    proc_macro_deps = [
+        "@raze__serde_derive__1_0_118//:serde_derive",
+    ],
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.3.3",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__atty__0_2_14//:atty",
+        "@raze__cast__0_2_3//:cast",
+        "@raze__clap__2_33_3//:clap",
+        "@raze__criterion_plot__0_4_3//:criterion_plot",
+        "@raze__csv__1_1_5//:csv",
+        "@raze__itertools__0_9_0//:itertools",
+        "@raze__lazy_static__1_4_0//:lazy_static",
+        "@raze__num_traits__0_2_14//:num_traits",
+        "@raze__oorandom__11_1_3//:oorandom",
+        "@raze__plotters__0_2_15//:plotters",
+        "@raze__rayon__1_5_0//:rayon",
+        "@raze__regex__1_4_2//:regex",
+        "@raze__serde__1_0_118//:serde",
+        "@raze__serde_cbor__0_11_1//:serde_cbor",
+        "@raze__serde_json__1_0_61//:serde_json",
+        "@raze__tinytemplate__1_2_0//:tinytemplate",
+        "@raze__walkdir__2_3_1//:walkdir",
+    ],
+)
+
+# Unsupported target "criterion_tests" with type "test" omitted

rules/rust/remote/BUILD.criterion-plot-0.4.3.bazel

@@ -0,0 +1,55 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "criterion_plot",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.4.3",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__cast__0_2_3//:cast",
+        "@raze__itertools__0_9_0//:itertools",
+    ],
+)

rules/rust/remote/BUILD.csv-1.1.5.bazel

@@ -0,0 +1,134 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "unencumbered",  # Unlicense from expression "Unlicense OR MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "bench" with type "bench" omitted
+
+# Unsupported target "cookbook-read-basic" with type "example" omitted
+
+# Unsupported target "cookbook-read-colon" with type "example" omitted
+
+# Unsupported target "cookbook-read-no-headers" with type "example" omitted
+
+# Unsupported target "cookbook-read-serde" with type "example" omitted
+
+# Unsupported target "cookbook-write-basic" with type "example" omitted
+
+# Unsupported target "cookbook-write-serde" with type "example" omitted
+
+# Unsupported target "tutorial-error-01" with type "example" omitted
+
+# Unsupported target "tutorial-error-02" with type "example" omitted
+
+# Unsupported target "tutorial-error-03" with type "example" omitted
+
+# Unsupported target "tutorial-error-04" with type "example" omitted
+
+# Unsupported target "tutorial-perf-alloc-01" with type "example" omitted
+
+# Unsupported target "tutorial-perf-alloc-02" with type "example" omitted
+
+# Unsupported target "tutorial-perf-alloc-03" with type "example" omitted
+
+# Unsupported target "tutorial-perf-core-01" with type "example" omitted
+
+# Unsupported target "tutorial-perf-serde-01" with type "example" omitted
+
+# Unsupported target "tutorial-perf-serde-02" with type "example" omitted
+
+# Unsupported target "tutorial-perf-serde-03" with type "example" omitted
+
+# Unsupported target "tutorial-pipeline-pop-01" with type "example" omitted
+
+# Unsupported target "tutorial-pipeline-search-01" with type "example" omitted
+
+# Unsupported target "tutorial-pipeline-search-02" with type "example" omitted
+
+# Unsupported target "tutorial-read-01" with type "example" omitted
+
+# Unsupported target "tutorial-read-delimiter-01" with type "example" omitted
+
+# Unsupported target "tutorial-read-headers-01" with type "example" omitted
+
+# Unsupported target "tutorial-read-headers-02" with type "example" omitted
+
+# Unsupported target "tutorial-read-serde-01" with type "example" omitted
+
+# Unsupported target "tutorial-read-serde-02" with type "example" omitted
+
+# Unsupported target "tutorial-read-serde-03" with type "example" omitted
+
+# Unsupported target "tutorial-read-serde-04" with type "example" omitted
+
+# Unsupported target "tutorial-read-serde-invalid-01" with type "example" omitted
+
+# Unsupported target "tutorial-read-serde-invalid-02" with type "example" omitted
+
+# Unsupported target "tutorial-setup-01" with type "example" omitted
+
+# Unsupported target "tutorial-write-01" with type "example" omitted
+
+# Unsupported target "tutorial-write-02" with type "example" omitted
+
+# Unsupported target "tutorial-write-delimiter-01" with type "example" omitted
+
+# Unsupported target "tutorial-write-serde-01" with type "example" omitted
+
+# Unsupported target "tutorial-write-serde-02" with type "example" omitted
+
+rust_library(
+    name = "csv",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "1.1.5",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__bstr__0_2_14//:bstr",
+        "@raze__csv_core__0_1_10//:csv_core",
+        "@raze__itoa__0_4_7//:itoa",
+        "@raze__ryu__1_0_5//:ryu",
+        "@raze__serde__1_0_118//:serde",
+    ],
+)
+
+# Unsupported target "tests" with type "test" omitted

rules/rust/remote/BUILD.csv-core-0.1.10.bazel

@@ -0,0 +1,57 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "unencumbered",  # Unlicense from expression "Unlicense OR MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "bench" with type "bench" omitted
+
+rust_library(
+    name = "csv_core",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+        "default",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.1.10",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__memchr__2_3_4//:memchr",
+    ],
+)

rules/rust/remote/BUILD.half-1.6.0.bazel

@@ -0,0 +1,57 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+# Unsupported target "convert" with type "bench" omitted
+
+rust_library(
+    name = "half",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "1.6.0",
+    # buildifier: leave-alone
+    deps = [
+    ],
+)
+
+# Unsupported target "version-numbers" with type "test" omitted

rules/rust/remote/BUILD.js-sys-0.3.46.bazel

@@ -0,0 +1,58 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "js_sys",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.3.46",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__wasm_bindgen__0_2_69//:wasm_bindgen",
+    ],
+)
+
+# Unsupported target "headless" with type "test" omitted
+
+# Unsupported target "wasm" with type "test" omitted

rules/rust/remote/BUILD.oorandom-11.1.3.bazel

@@ -0,0 +1,53 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "oorandom",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "11.1.3",
+    # buildifier: leave-alone
+    deps = [
+    ],
+)

rules/rust/remote/BUILD.plotters-0.2.15.bazel

@@ -0,0 +1,105 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "benchmark" with type "bench" omitted
+
+# Unsupported target "animation" with type "example" omitted
+
+# Unsupported target "area-chart" with type "example" omitted
+
+# Unsupported target "blit-bitmap" with type "example" omitted
+
+# Unsupported target "boxplot" with type "example" omitted
+
+# Unsupported target "chart" with type "example" omitted
+
+# Unsupported target "console" with type "example" omitted
+
+# Unsupported target "errorbar" with type "example" omitted
+
+# Unsupported target "histogram" with type "example" omitted
+
+# Unsupported target "mandelbrot" with type "example" omitted
+
+# Unsupported target "matshow" with type "example" omitted
+
+# Unsupported target "normal-dist" with type "example" omitted
+
+# Unsupported target "normal-dist2" with type "example" omitted
+
+# Unsupported target "relative_size" with type "example" omitted
+
+# Unsupported target "sierpinski" with type "example" omitted
+
+# Unsupported target "slc-temp" with type "example" omitted
+
+# Unsupported target "snowflake" with type "example" omitted
+
+# Unsupported target "stock" with type "example" omitted
+
+# Unsupported target "two-scales" with type "example" omitted
+
+rust_library(
+    name = "plotters",
+    srcs = glob(["**/*.rs"]),
+    aliases = {
+    },
+    crate_features = [
+        "area_series",
+        "line_series",
+        "svg",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.2.15",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__num_traits__0_2_14//:num_traits",
+    ] + selects.with_or({
+        # cfg(not(target_arch = "wasm32"))
+        (
+            "@io_bazel_rules_rust//rust/platform:x86_64-apple-darwin",
+            "@io_bazel_rules_rust//rust/platform:x86_64-unknown-linux-gnu",
+        ): [
+        ],
+        "//conditions:default": [],
+    }),
+)

rules/rust/remote/BUILD.regex-automata-0.1.9.bazel

@@ -0,0 +1,56 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "unencumbered",  # Unlicense from expression "Unlicense OR MIT"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "regex_automata",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.1.9",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__byteorder__1_3_4//:byteorder",
+    ],
+)
+
+# Unsupported target "default" with type "test" omitted

rules/rust/remote/BUILD.same-file-1.0.6.bazel

@@ -0,0 +1,57 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "unencumbered",  # Unlicense from expression "Unlicense OR MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "is_same_file" with type "example" omitted
+
+# Unsupported target "is_stderr" with type "example" omitted
+
+rust_library(
+    name = "same_file",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "1.0.6",
+    # buildifier: leave-alone
+    deps = [
+    ],
+)

rules/rust/remote/BUILD.serde_cbor-0.11.1.bazel

@@ -0,0 +1,77 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+# Unsupported target "readme" with type "example" omitted
+
+# Unsupported target "tags" with type "example" omitted
+
+rust_library(
+    name = "serde_cbor",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+        "default",
+        "std",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.11.1",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__half__1_6_0//:half",
+        "@raze__serde__1_0_118//:serde",
+    ],
+)
+
+# Unsupported target "bennofs" with type "test" omitted
+
+# Unsupported target "canonical" with type "test" omitted
+
+# Unsupported target "de" with type "test" omitted
+
+# Unsupported target "enum" with type "test" omitted
+
+# Unsupported target "ser" with type "test" omitted
+
+# Unsupported target "std_types" with type "test" omitted
+
+# Unsupported target "tags" with type "test" omitted
+
+# Unsupported target "value" with type "test" omitted

rules/rust/remote/BUILD.tinytemplate-1.2.0.bazel

@@ -0,0 +1,57 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # Apache-2.0 from expression "Apache-2.0 OR MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "benchmarks" with type "bench" omitted
+
+rust_library(
+    name = "tinytemplate",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2015",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "1.2.0",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__serde__1_0_118//:serde",
+        "@raze__serde_json__1_0_61//:serde_json",
+    ],
+)

rules/rust/remote/BUILD.walkdir-2.3.1.bazel

@@ -0,0 +1,54 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "unencumbered",  # Unlicense from expression "Unlicense OR MIT"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "walkdir",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "2.3.1",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__same_file__1_0_6//:same_file",
+    ],
+)

rules/rust/remote/BUILD.web-sys-0.3.46.bazel

@@ -0,0 +1,67 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "notice",  # MIT from expression "MIT OR Apache-2.0"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "web_sys",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+        "CanvasRenderingContext2d",
+        "Document",
+        "DomRect",
+        "DomRectReadOnly",
+        "Element",
+        "EventTarget",
+        "HtmlCanvasElement",
+        "HtmlElement",
+        "Node",
+        "Window",
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.3.46",
+    # buildifier: leave-alone
+    deps = [
+        "@raze__js_sys__0_3_46//:js_sys",
+        "@raze__wasm_bindgen__0_2_69//:wasm_bindgen",
+    ],
+)
+
+# Unsupported target "wasm" with type "test" omitted

rules/rust/remote/BUILD.winapi-util-0.1.5.bazel

@@ -0,0 +1,53 @@
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load(
+    "@io_bazel_rules_rust//rust:rust.bzl",
+    "rust_binary",
+    "rust_library",
+    "rust_test",
+)
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+package(default_visibility = [
+    # Public for visibility by "@raze__crate__version//" targets.
+    #
+    # Prefer access through "//rules/rust", which limits external
+    # visibility to explicit Cargo.toml dependencies.
+    "//visibility:public",
+])
+
+licenses([
+    "unencumbered",  # Unlicense from expression "Unlicense OR MIT"
+])
+
+# Generated Targets
+
+rust_library(
+    name = "winapi_util",
+    srcs = glob(["**/*.rs"]),
+    crate_features = [
+    ],
+    crate_root = "src/lib.rs",
+    crate_type = "lib",
+    data = [],
+    edition = "2018",
+    rustc_flags = [
+        "--cap-lints=allow",
+    ],
+    tags = [
+        "cargo-raze",
+        "manual",
+    ],
+    version = "0.1.5",
+    # buildifier: leave-alone
+    deps = [
+    ],
+)