MarcoBuster/hyperboria
1
0
Fork 0
mirror of https://github.com/nexus-stc/hyperboria synced 2024-11-26 21:16:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

- [nexus] Development

Browse Source 
GitOrigin-RevId: d17172248b2b9b056335f96d7fcfa155c281f10b
This commit is contained in:
the-superpirate 2022-09-13 18:50:31 +03:00
parent 971cdf6245
commit 030743860e
4 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
library/aiopostgres/pool_holder.py
View File

@ -92,7 +92,6 @@ class AioPostgresPoolHolder(AioThing):
row_factory=tuple_row, row_factory=tuple_row,
cursor_name: Optional[str] = None, cursor_name: Optional[str] = None,
itersize: Optional[int] = None, itersize: Optional[int] = None,
statement_timeout: Optional[int] = None,
): ):
if not self.pool: if not self.pool:
raise RuntimeError('AioPostgresPoolHolder has not been started') raise RuntimeError('AioPostgresPoolHolder has not been started')
@ -100,9 +99,7 @@ class AioPostgresPoolHolder(AioThing):
async with conn.cursor(name=cursor_name, row_factory=row_factory) as cur: async with conn.cursor(name=cursor_name, row_factory=row_factory) as cur:
if itersize is not None: if itersize is not None:
cur.itersize = itersize cur.itersize = itersize
await cur.execute(stmt + ';' if statement_timeout else '', values) await cur.execute(stmt, values)
if statement_timeout:
await cur.execute(f'SET statement_timeout = {statement_timeout};')
async for row in cur: async for row in cur:
yield row yield row

5
nexus/ingest/jobs/postgres.py
View File

@ -72,10 +72,10 @@ class PostgresJob(BaseJob):
async def iterator(self) -> AsyncIterable[Any]: async def iterator(self) -> AsyncIterable[Any]:
session_id = generate_request_id() session_id = generate_request_id()
await self.init_tables(session_id) # await self.init_tables(session_id)
if self.batch_size: if self.batch_size:
loaded = True loaded = True
current = 0 current = 168_000_000
while loaded: while loaded:
loaded = False loaded = False
sql = self.sql.format(left=current, right=current + self.batch_size) sql = self.sql.format(left=current, right=current + self.batch_size)
@ -85,7 +85,6 @@ class PostgresJob(BaseJob):
# Mandatory for server side cursor # Mandatory for server side cursor
cursor_name='nexus_ingest_cursor', cursor_name='nexus_ingest_cursor',
itersize=50_000, itersize=50_000,
statement_timeout=3600 * 2,
): ):
loaded = True loaded = True
yield row yield row

1
rules/python/requirements-lock.txt
View File

@ -55,6 +55,7 @@ h11==0.13.0
idna==3.3 idna==3.3
iniconfig==1.1.1 iniconfig==1.1.1
isort==5.10.1 isort==5.10.1
izihawa-configurator==1.0.0
izihawa-nlptools==1.1.9 izihawa-nlptools==1.1.9
izihawa-types==0.1.3 izihawa-types==0.1.3
izihawa-utils==1.0.7 izihawa-utils==1.0.7

4
rules/python/requirements.txt
View File

@ -31,8 +31,10 @@ fire
flake8 flake8
grpcio grpcio
isort isort
izihawa-utils izihawa-configurator
izihawa-nlptools
izihawa-types izihawa-types
izihawa-utils
jinja2 jinja2
lemminflect lemminflect
lightgbm lightgbm