diff --git a/example/emscripten/build-openssl.sh b/example/emscripten/build-openssl.sh new file mode 100755 index 00000000..5ef3eb2f --- /dev/null +++ b/example/emscripten/build-openssl.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +OPENSSL=OpenSSL_1_1_0j +if [ ! -f $OPENSSL.tar.gz ]; then + echo "Download openssl" + wget https://github.com/openssl/openssl/archive/$OPENSSL.tar.gz +fi +rm -rf ./$OPENSSL +tar xzf $OPENSSL.tar.gz || exit 1 +cd openssl-$OPENSSL + +emconfigure ./Configure linux-generic32 no-shared +sed -i bak 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile +emmake make depend -s || exit 1 +emmake make -s || exit 1 + +rm -rf ../build/crypto || exit 1 +mkdir -p ../build/crypto/lib || exit 1 +cp libcrypto.a libssl.a ../build/crypto/lib/ || exit 1 +cp -r include ../build/crypto/ || exit 1 +cd .. diff --git a/example/emscripten/build-tdlib.sh b/example/emscripten/build-tdlib.sh new file mode 100755 index 00000000..09f6fb5d --- /dev/null +++ b/example/emscripten/build-tdlib.sh @@ -0,0 +1,37 @@ +mkdir -p build/generate +mkdir -p build/asmjs +mkdir -p build/wasm + +TD_ROOT=$(realpath ../../) +#OPENSSL_OPTIONS="-DOPENSSL_ROOT=$TD_ROOT/third_party/crypto/emscripten" +OPENSSL_ROOT=$(realpath ./build/crypto/) +OPENSSL_CRYPTO_LIBRARY=$OPENSSL_ROOT/lib/libcrypto.a +OPENSSL_SSL_LIBRARY=$OPENSSL_ROOT/lib/libssl.a + +OPENSSL_OPTIONS="-DOPENSSL_FOUND=1 \ + -DOPENSSL_ROOT_DIR=\"$OPENSSL_ROOT\" \ + -DOPENSSL_INCLUDE_DIR=\"$OPENSSL_ROOT/include\" \ + -DOPENSSL_CRYPTO_LIBRARY=\"$OPENSSL_CRYPTO_LIBRARY\" \ + -DOPENSSL_SSL_LIBRARY=\"$OPENSSL_SSL_LIBRARY\" \ + -DOPENSSL_LIBRARIES=\"$OPENSSL_SSL_LIBRARY;$OPENSSL_CRYPTO_LIBRARY\" \ + -DOPENSSL_VERSION=\"1.1.0j\"" + +pushd . +cd build/wasm +eval emconfigure cmake $TD_ROOT -GNinja $OPENSSL_OPTIONS +popd + +pushd . +cd build/asmjs +eval emconfigure cmake $TD_ROOT -GNinja -DASMJS=1 $OPENSSL_OPTIONS +popd + +pushd . +cd build/generate +cmake $TD_ROOT -GNinja +popd + +cmake --build build/generate -j --target prepare_cross_compiling +cmake --build build/wasm -j --target td_wasm +cmake --build build/asmjs -j --target td_asmjs + diff --git a/example/emscripten/build-tdweb.sh b/example/emscripten/build-tdweb.sh new file mode 100755 index 00000000..34f53347 --- /dev/null +++ b/example/emscripten/build-tdweb.sh @@ -0,0 +1,4 @@ +pushd . +cd tdweb +npm install +npm run build diff --git a/example/emscripten/copy-tdlib.sh b/example/emscripten/copy-tdlib.sh new file mode 100755 index 00000000..117cf49a --- /dev/null +++ b/example/emscripten/copy-tdlib.sh @@ -0,0 +1,4 @@ +dest=tdweb/src/prebuilt/release/ +mkdir -p $dest +cp build/wasm/td_wasm.{js,wasm} $dest +cp build/asmjs/td_asmjs.js{,.mem} $dest diff --git a/example/emscripten/tdweb/package.json b/example/emscripten/tdweb/package.json new file mode 100644 index 00000000..e188d14f --- /dev/null +++ b/example/emscripten/tdweb/package.json @@ -0,0 +1,86 @@ +{ + "name": "@arseny30/tdweb", + "version": "0.2.23", + "description": "Javascript interface for TDLib (telegram library)", + "main": "dist/tdweb.js", + "files": [ + "dist" + ], + "scripts": { + "precommit": "lint-staged", + "build": "node --max_old_space_size=8192 node_modules/.bin/webpack ", + "start": "webpack-dev-server --open" + }, + "keywords": [ + "telegram" + ], + "author": "Arseny Smirnov", + "license": "MIT", + "devDependencies": { + "babel-core": "^6.26.3", + "babel-eslint": "^7.2.3", + "babel-loader": "^7.1.5", + "babel-plugin-syntax-dynamic-import": "^6.18.0", + "babel-plugin-transform-runtime": "^6.23.0", + "babel-preset-env": "^1.7.0", + "clean-webpack-plugin": "^0.1.19", + "eslint": "^4.19.1", + "eslint-config-react-app": "^2.1.0", + "eslint-loader": "^1.9.0", + "eslint-plugin-flowtype": "^2.50.1", + "eslint-plugin-import": "^2.14.0", + "eslint-plugin-jsx-a11y": "^5.1.1", + "eslint-plugin-react": "^7.11.1", + "exports-loader": "^0.6.4", + "file-loader": "^1.1.11", + "html-webpack-plugin": "^2.30.1", + "husky": "^0.14.3", + "lint-staged": "^4.3.0", + "prettier": "^1.14.2", + "script-loader": "^0.7.2", + "uglifyjs-webpack-plugin": "^1.3.0", + "webpack": "^3.12.0", + "webpack-dev-server": "^2.11.3", + "worker-loader": "^1.1.1" + }, + "lint-staged": { + "webpack.config.json": [ + "prettier --single-quote --write", + "git add" + ], + "package.json": [ + "prettier --single-quote --write", + "git add" + ], + "src/*.{js,jsx,json,css}": [ + "prettier --single-quote --write", + "git add" + ] + }, + "dependencies": { + "babel-runtime": "^6.26.0", + "detect-browser": "^2.5.1", + "localforage": "^1.7.2", + "uuid": "^3.3.2" + }, + "babel": { + "presets": [ + "env" + ], + "plugins": [ + "syntax-dynamic-import", + "transform-runtime" + ] + }, + "eslintConfig": { + "extends": "eslint-config-react-app", + "env": { + "worker": true, + "node": true, + "browser": true + }, + "globals": { + "WebAssembly": true + } + } +} diff --git a/example/emscripten/tdweb/src/index.js b/example/emscripten/tdweb/src/index.js new file mode 100644 index 00000000..0ac70b01 --- /dev/null +++ b/example/emscripten/tdweb/src/index.js @@ -0,0 +1,212 @@ +import MyWorker from './worker.js'; +import './third_party/broadcastchannel.js'; +import uuid4 from 'uuid/v4'; +import log from './logger.js'; + +const sleep = ms => new Promise(res => setTimeout(res, ms)); + +class TdClient { + constructor(options) { + log.setVerbosity(options.jsVerbosity); + this.worker = new MyWorker(); + var self = this; + this.worker.onmessage = function(e) { + let response = e.data; + log.info( + 'receive from worker: ', + JSON.parse( + JSON.stringify(response, (key, value) => { + if (key === 'arr') { + return undefined; + } + return value; + }) + ) + ); + if ('@extra' in response) { + var query_id = response['@extra'].query_id; + var [resolve, reject] = self.query_callbacks.get(query_id); + self.query_callbacks.delete(query_id); + if ('@old_extra' in response['@extra']) { + response['@extra'] = response['@extra']['@old_extra']; + } + if (resolve) { + if (response['@type'] === 'error') { + reject(response); + } else { + resolve(response); + } + } + } else { + if (response['@type'] === 'inited') { + self.onInited(); + return; + } + if ( + response['@type'] === 'updateAuthorizationState' && + response.authorization_state['@type'] === 'authorizationStateClosed' + ) { + self.onClosed(); + } + self.onUpdate(response); + } + }; + this.query_id = 0; + this.query_callbacks = new Map(); + this.worker.postMessage({ '@type': 'init', options: options }); + this.closeOtherClients(options); + } + + onBroadcastMessage(e) { + var message = e.data; + log.info('got broadcast message: ', message); + if (message.isBackground && !this.isBackground) { + // continue + } else if ( + (!message.isBackground && this.isBackground) || + message.timestamp > this.timestamp + ) { + this.close(); + return; + } + if (message.state === 'closed') { + this.waitSet.delete(message.uid); + if (this.waitSet.size === 0) { + log.info('onWaitSetEmpty'); + this.onWaitSetEmpty(); + this.onWaitSetEmpty = () => {}; + } + } else { + this.waitSet.add(message.uid); + if (message.state !== 'closing') { + this.postState(); + } + } + } + + postState() { + let state = { + id: this.uid, + state: this.state, + timestamp: this.timestamp, + isBackground: this.isBackground + }; + log.info('Post state: ', state); + this.channel.postMessage(state); + } + + onWaitSetEmpty() { + // nop + } + + onInited() { + this.isInited = true; + this.doSendStart(); + } + sendStart() { + this.wantSendStart = true; + this.doSendStart(); + } + + doSendStart() { + if (!this.isInited || !this.wantSendStart || this.state !== 'start') { + return; + } + this.wantSendStart = false; + this.state = 'active'; + let query = { '@type': 'start' }; + log.info('send to worker: ', query); + this.worker.postMessage(query); + } + + onClosed() { + this.isClosing = true; + this.worker.terminate(); + log.info('worker is terminated'); + this.state = 'closed'; + this.postState(); + } + + close() { + if (this.isClosing) { + return; + } + this.isClosing = true; + + log.info('close state: ', this.state); + + if (this.state === 'start') { + this.onClosed(); + this.onUpdate({ + '@type': 'updateAuthorizationState', + authorization_state: { + '@type': 'authorizationStateClosed' + } + }); + return; + } + + let query = { '@type': 'close' }; + log.info('send to worker: ', query); + this.worker.postMessage(query); + + this.state = 'closing'; + this.postState(); + } + + async closeOtherClients(options) { + this.uid = uuid4(); + this.state = 'start'; + this.isBackground = !!options.isBackground; + this.timestamp = Date.now(); + this.waitSet = new Set(); + + log.info('close other clients'); + let prefix = options.prefix || 'tdlib'; + this.channel = new BroadcastChannel(prefix); + + this.postState(); + + var self = this; + this.channel.onmessage = message => { + self.onBroadcastMessage(message); + }; + + await sleep(300); + if (this.waitSet.size !== 0) { + await new Promise(resolve => { + self.onWaitSetEmpty = resolve; + }); + } + this.sendStart(); + } + + onUpdate(response) { + log.info('ignore onUpdate'); + //nop + } + + send(query) { + this.query_id++; + if (query['@extra']) { + query['@extra'] = { + '@old_extra': JSON.parse(JSON.stringify(query.extra)), + query_id: this.query_id + }; + } else { + query['@extra'] = { + query_id: this.query_id + }; + } + if (query['@type'] === 'setJsVerbosity') { + log.setVerbosity(query.verbosity); + } + + log.info('send to worker: ', query); + this.worker.postMessage(query); + return new Promise((resolve, reject) => { + this.query_callbacks.set(this.query_id, [resolve, reject]); + }); + } +} +export default TdClient; diff --git a/example/emscripten/tdweb/src/logger.js b/example/emscripten/tdweb/src/logger.js new file mode 100644 index 00000000..7e76a6aa --- /dev/null +++ b/example/emscripten/tdweb/src/logger.js @@ -0,0 +1,47 @@ +class Logger { + constructor() { + this.setVerbosity('WARNING'); + } + debug(...str) { + if (this.checkVerbosity(4)) { + console.log(...str); + } + } + log(...str) { + if (this.checkVerbosity(4)) { + console.log(...str); + } + } + info(...str) { + if (this.checkVerbosity(3)) { + console.info(...str); + } + } + warn(...str) { + if (this.checkVerbosity(2)) { + console.warn(...str); + } + } + error(...str) { + if (this.checkVerbosity(1)) { + console.error(...str); + } + } + setVerbosity(level, default_level = 'info') { + if (level === undefined) { + level = default_level; + } + if (typeof level === 'string') { + level = + { ERROR: 1, WARNINIG: 2, INFO: 3, LOG: 4, DEBUG: 4 }[ + level.toUpperCase() + ] || 2; + } + this.level = level; + } + checkVerbosity(level) { + return this.level >= level; + } +} +let log = new Logger(); +export default log; diff --git a/example/emscripten/tdweb/src/third_party/broadcastchannel.js b/example/emscripten/tdweb/src/third_party/broadcastchannel.js new file mode 100644 index 00000000..a3c46629 --- /dev/null +++ b/example/emscripten/tdweb/src/third_party/broadcastchannel.js @@ -0,0 +1,83 @@ +// from https://gist.github.com/inexorabletash/52f437d1451d12145264 +(function(global) { + var channels = []; + + function BroadcastChannel(channel) { + var $this = this; + channel = String(channel); + + var id = '$BroadcastChannel$' + channel + '$'; + + channels[id] = channels[id] || []; + channels[id].push(this); + + this._name = channel; + this._id = id; + this._closed = false; + this._mc = new MessageChannel(); + this._mc.port1.start(); + this._mc.port2.start(); + + global.addEventListener('storage', function(e) { + if (e.storageArea !== global.localStorage) return; + if (e.newValue === null) return; + if (e.key.substring(0, id.length) !== id) return; + var data = JSON.parse(e.newValue); + $this._mc.port2.postMessage(data); + }); + } + + BroadcastChannel.prototype = { + // BroadcastChannel API + get name() { return this._name; }, + postMessage: function(message) { + var $this = this; + if (this._closed) { + var e = new Error(); + e.name = 'InvalidStateError'; + throw e; + } + var value = JSON.stringify(message); + + // Broadcast to other contexts via storage events... + var key = this._id + String(Date.now()) + '$' + String(Math.random()); + global.localStorage.setItem(key, value); + setTimeout(function() { global.localStorage.removeItem(key); }, 500); + + // Broadcast to current context via ports + channels[this._id].forEach(function(bc) { + if (bc === $this) return; + bc._mc.port2.postMessage(JSON.parse(value)); + }); + }, + close: function() { + if (this._closed) return; + this._closed = true; + this._mc.port1.close(); + this._mc.port2.close(); + + var index = channels[this._id].indexOf(this); + channels[this._id].splice(index, 1); + }, + + // EventTarget API + get onmessage() { return this._mc.port1.onmessage; }, + set onmessage(value) { this._mc.port1.onmessage = value; }, + addEventListener: function(type, listener /*, useCapture*/) { + return this._mc.port1.addEventListener.apply(this._mc.port1, arguments); + }, + removeEventListener: function(type, listener /*, useCapture*/) { + return this._mc.port1.removeEventListener.apply(this._mc.port1, arguments); + }, + dispatchEvent: function(event) { + return this._mc.port1.dispatchEvent.apply(this._mc.port1, arguments); + } + }; + + if (global.BroadcastChannel) { + console.log("already has native BroadcastChannel"); + } else { + global.BroadcastChannel = BroadcastChannel; + console.log("use polyfill for BroadcastChannel"); + } +}(window.top)); diff --git a/example/emscripten/tdweb/src/wasm-utils.js b/example/emscripten/tdweb/src/wasm-utils.js new file mode 100644 index 00000000..b364cc1e --- /dev/null +++ b/example/emscripten/tdweb/src/wasm-utils.js @@ -0,0 +1,121 @@ +// 1. +++ fetchAndInstantiate() +++ // + +// This library function fetches the wasm module at 'url', instantiates it with +// the given 'importObject', and returns the instantiated object instance + +export function instantiateStreaming(url, importObject) { + return WebAssembly.instantiateStreaming(fetch(url), importObject).then( + results => results.instance + ); +} +export function fetchAndInstantiate(url, importObject) { + return fetch(url) + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, importObject)) + .then(results => results.instance); +} + +// 2. +++ instantiateCachedURL() +++ // + +// This library function fetches the wasm Module at 'url', instantiates it with +// the given 'importObject', and returns a Promise resolving to the finished +// wasm Instance. Additionally, the function attempts to cache the compiled wasm +// Module in IndexedDB using 'url' as the key. The entire site's wasm cache (not +// just the given URL) is versioned by dbVersion and any change in dbVersion on +// any call to instantiateCachedURL() will conservatively clear out the entire +// cache to avoid stale modules. +export function instantiateCachedURL(dbVersion, url, importObject) { + const dbName = 'wasm-cache'; + const storeName = 'wasm-cache'; + + // This helper function Promise-ifies the operation of opening an IndexedDB + // database and clearing out the cache when the version changes. + function openDatabase() { + return new Promise((resolve, reject) => { + var request = indexedDB.open(dbName, dbVersion); + request.onerror = reject.bind(null, 'Error opening wasm cache database'); + request.onsuccess = () => { + resolve(request.result); + }; + request.onupgradeneeded = event => { + var db = request.result; + if (db.objectStoreNames.contains(storeName)) { + console.log(`Clearing out version ${event.oldVersion} wasm cache`); + db.deleteObjectStore(storeName); + } + console.log(`Creating version ${event.newVersion} wasm cache`); + db.createObjectStore(storeName); + }; + }); + } + + // This helper function Promise-ifies the operation of looking up 'url' in the + // given IDBDatabase. + function lookupInDatabase(db) { + return new Promise((resolve, reject) => { + var store = db.transaction([storeName]).objectStore(storeName); + var request = store.get(url); + request.onerror = reject.bind(null, `Error getting wasm module ${url}`); + request.onsuccess = event => { + if (request.result) resolve(request.result); + else reject(`Module ${url} was not found in wasm cache`); + }; + }); + } + + // This helper function fires off an async operation to store the given wasm + // Module in the given IDBDatabase. + function storeInDatabase(db, module) { + var store = db.transaction([storeName], 'readwrite').objectStore(storeName); + var request = store.put(module, url); + request.onerror = err => { + console.log(`Failed to store in wasm cache: ${err}`); + }; + request.onsuccess = err => { + console.log(`Successfully stored ${url} in wasm cache`); + }; + } + + // This helper function fetches 'url', compiles it into a Module, + // instantiates the Module with the given import object. + function fetchAndInstantiate() { + return fetch(url) + .then(response => response.arrayBuffer()) + .then(buffer => WebAssembly.instantiate(buffer, importObject)); + } + + // With all the Promise helper functions defined, we can now express the core + // logic of an IndexedDB cache lookup. We start by trying to open a database. + return openDatabase().then( + db => { + // Now see if we already have a compiled Module with key 'url' in 'db': + return lookupInDatabase(db).then( + module => { + // We do! Instantiate it with the given import object. + console.log(`Found ${url} in wasm cache`); + return WebAssembly.instantiate(module, importObject); + }, + errMsg => { + // Nope! Compile from scratch and then store the compiled Module in 'db' + // with key 'url' for next time. + console.log(errMsg); + return fetchAndInstantiate().then(results => { + try { + storeInDatabase(db, results.module); + } catch (e) { + console.log('Failed to store module into db'); + } + return results.instance; + }); + } + ); + }, + errMsg => { + // If opening the database failed (due to permissions or quota), fall back + // to simply fetching and compiling the module and don't try to store the + // results. + console.log(errMsg); + return fetchAndInstantiate().then(results => results.instance); + } + ); +} diff --git a/example/emscripten/tdweb/src/worker.js b/example/emscripten/tdweb/src/worker.js new file mode 100644 index 00000000..1baf9a24 --- /dev/null +++ b/example/emscripten/tdweb/src/worker.js @@ -0,0 +1,543 @@ +import localforage from 'localforage'; +import log from './logger.js'; +import { + instantiateCachedURL, + /*fetchAndInstantiate,*/ instantiateStreaming +} from './wasm-utils.js'; + +import td_wasm_release from './prebuilt/release/td_wasm.wasm'; + +// Uncomment for asmjs support +//import td_asmjs_mem_release from './prebuilt/release/td_asmjs.js.mem'; + +import { detect } from 'detect-browser'; +const browser = detect(); +const tdlibVersion = 5; + +async function loadTdLibWasm(useStreaming) { + let Module = await import('./prebuilt/release/td_wasm.js'); + log.info('got td_wasm.js'); + let td_wasm = td_wasm_release; + let TdModule = new Promise((resolve, reject) => + Module({ + onRuntimeInitialized: () => { + log.info('runtime intialized'); + }, + instantiateWasm: (imports, successCallback) => { + log.info('start instantiateWasm'); + let next = instance => { + log.info('finish instantiateWasm'); + successCallback(instance); + }; + if (useStreaming) { + instantiateStreaming(td_wasm, imports).then(next); + } else { + instantiateCachedURL(tdlibVersion, td_wasm, imports).then(next); + } + return {}; + }, + ENVIROMENT: 'WORKER' + }).then(m => { + delete m.then; + resolve(m); + }) + ); + + return TdModule; +} + +// Uncomment for asmjs support +//async function loadTdLibAsmjs() { + //let Module = await import('./prebuilt/release/td_asmjs.js'); + //console.log('got td_wasm.js'); + //let fromFile = 'td_asmjs.js.mem'; + //let toFile = td_asmjs_mem_release; + //let TdModule = new Promise((resolve, reject) => + //Module({ + //onRuntimeInitialized: () => { + //console.log('runtime intialized'); + //}, + //locateFile: name => { + //if (name === fromFile) { + //return toFile; + //} + //return name; + //}, + //ENVIROMENT: 'WORKER' + //}).then(m => { + //delete m.then; + //resolve(m); + //}) + //); + + //return TdModule; +//} + +async function loadTdLib(mode) { +// Uncomment for asmjs support + //if (mode === 'asmjs') { + //return loadTdLibAsmjs(); + //} + return loadTdLibWasm(mode !== 'wasm'); +} + +class OutboundFileSystem { + constructor(root, FS) { + this.root = root; + this.nextFileId = 0; + this.FS = FS; + this.files = new Set(); + FS.mkdir(root); + } + blobToPath(blob, name) { + var dir = this.root + '/' + this.nextFileId; + if (!name) { + name = 'blob'; + } + this.nextFileId++; + this.FS.mkdir(dir); + this.FS.mount( + this.FS.filesystems.WORKERFS, + { + blobs: [{ name: name, data: blob }] + }, + dir + ); + let path = dir + '/' + name; + this.files.add(path); + return path; + } + + forgetPath(path) { + if (this.files.has(path)) { + this.FS.unmount(path); + this.files.delete(path); + } + } +} + +class InboundFileSystem { + static async create(dbName, root, FS) { + try { + let ifs = new InboundFileSystem(); + ifs.root = root; + ifs.FS = FS; + FS.mkdir(root); + + ifs.store = localforage.createInstance({ + name: dbName + }); + let keys = await ifs.store.keys(); + + ifs.pids = new Set(keys); + return ifs; + } catch (e) { + log.error('Failed to init Inbound FileSystem: ', e); + } + } + + has(pid) { + return this.pids.has(pid); + } + + forget(pid) { + this.pids.delete(pid); + } + + async persist(pid, path) { + var arr; + try { + arr = this.FS.readFile(path); + await this.store.setItem(pid, new Blob([arr])); + this.pids.add(pid); + this.FS.unlink(path); + } catch (e) { + log.error('Failed persist ' + path + ' ', e); + } + return arr; + } +} + +class DbFileSystem { + static async create(root, FS, readOnly = false) { + try { + let dbfs = new DbFileSystem(); + dbfs.root = root; + dbfs.FS = FS; + dbfs.syncfs_total_time = 0; + dbfs.readOnly = readOnly; + FS.mkdir(root); + FS.mount(FS.filesystems.IDBFS, {}, root); + + await new Promise((resolve, reject) => { + FS.syncfs(true, err => { + resolve(); + }); + }); + + dbfs.syncfsInterval = setInterval(() => { + dbfs.sync(); + }, 5000); + return dbfs; + } catch (e) { + log.error('Failed to init DbFileSystem: ', e); + } + } + async sync() { + if (this.readOnly) { + return; + } + let start = performance.now(); + await new Promise((resolve, reject) => { + this.FS.syncfs(false, () => { + let syncfs_time = (performance.now() - start) / 1000; + this.syncfs_total_time += syncfs_time; + log.debug('SYNC: ' + syncfs_time); + log.debug('SYNC total: ' + this.syncfs_total_time); + resolve(); + }); + }); + } + async close() { + clearInterval(this.syncfsInterval); + await this.sync(); + } +} + +class TdFileSystem { + static async create(prefix, FS, readOnly = false) { + try { + let tdfs = new TdFileSystem(); + tdfs.prefix = prefix; + tdfs.FS = FS; + FS.mkdir(prefix); + + //WORKERFS. Temporary stores Blobs for outbound files + tdfs.outboundFileSystem = new OutboundFileSystem( + prefix + '/outboundfs', + FS + ); + + //MEMFS. Store to IDB and delete files as soon as possible + let inboundFileSystem = InboundFileSystem.create( + prefix, + prefix + '/inboundfs', + FS + ); + + //IDBFS. MEMFS which is flushed to IDB from time to time + let dbFileSystem = DbFileSystem.create(prefix + '/dbfs', FS, readOnly); + + tdfs.inboundFileSystem = await inboundFileSystem; + tdfs.dbFileSystem = await dbFileSystem; + return tdfs; + } catch (e) { + log.error('Failed to init TdFileSystem: ', e); + } + } +} + +class TdClient { + constructor(callback) { + log.info('Start worker'); + this.pendingQueries = []; + this.isPending = true; + this.callback = callback; + this.wasInit = false; + } + + async init(options) { + if (this.wasInit) { + return; + } + log.setVerbosity(options.jsVerbosity); + this.wasInit = true; + + options = options || {}; + let mode = 'wasm'; + if (browser && (browser.name === 'chrome' || browser.name === 'safari')) { + mode = 'asmjs'; + } + mode = options.mode || mode; + + this.TdModule = await loadTdLib(mode); + log.info('got TdModule'); + this.td_functions = { + td_create: this.TdModule.cwrap('td_create', 'number', []), + td_destroy: this.TdModule.cwrap('td_destroy', null, ['number']), + td_send: this.TdModule.cwrap('td_send', null, ['number', 'string']), + td_execute: this.TdModule.cwrap('td_execute', 'string', [ + 'number', + 'string' + ]), + td_receive: this.TdModule.cwrap('td_receive', 'string', ['number']), + td_set_verbosity: verbosity => { + this.td_functions.td_execute( + 0, + JSON.stringify({ + '@type': 'setLogVerbosityLevel', + new_verbosity_level: verbosity + }) + ); + }, + td_get_timeout: this.TdModule.cwrap('td_get_timeout', 'number', []) + }; + this.FS = this.TdModule.FS; + this.TdModule['websocket']['on']('error', error => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('open', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('listen', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('connection', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('message', fd => { + this.scheduleReceiveSoon(); + }); + this.TdModule['websocket']['on']('close', fd => { + this.scheduleReceiveSoon(); + }); + + // wait till it is allowed to start + this.callback({ '@type': 'inited' }); + var self = this; + await new Promise(resolve => { + self.onStart = resolve; + }); + this.isStarted = true; + + log.info('may start now'); + if (this.isClosing) { + return; + } + let prefix = options.prefix || 'tdlib'; + log.info('FS start init'); + this.tdfs = await TdFileSystem.create( + '/' + prefix, + this.FS, + options.readOnly + ); + log.info('FS inited'); + + // no async initialization after this point + if (options.verbosity === undefined) { + options.verbosity = 5; + } + this.td_functions.td_set_verbosity(options.verbosity); + this.client = this.td_functions.td_create(); + + this.savingFiles = new Map(); + this.flushPendingQueries(); + + this.receive(); + //setInterval(()=>this.receive(), 100); + } + + prepareQueryRecursive(query) { + if (query['@type'] === 'inputFileBlob') { + return { + '@type': 'inputFileLocal', + path: this.tdfs.outboundFileSystem.blobToPath(query.blob, query.name) + }; + } + for (var key in query) { + let field = query[key]; + if (field && typeof field === 'object') { + query[key] = this.prepareQueryRecursive(field); + } + } + return query; + } + + prepareQuery(query) { + if (query['@type'] === 'setTdlibParameters') { + query.parameters.database_directory = this.tdfs.dbFileSystem.root; + query.parameters.files_directory = this.tdfs.inboundFileSystem.root; + } + return this.prepareQueryRecursive(query); + } + + onStart() { + //nop + log.info('ignore on_start'); + } + + send(query) { + if (this.wasFatalError || this.isClosing) { + return; + } + if (query['@type'] === 'init') { + this.init(query.options); + return; + } + if (query['@type'] === 'start') { + log.info('on_start'); + this.onStart(); + return; + } + if (query['@type'] === 'setJsVerbosity') { + log.setVerbosity(query.verbosity); + return; + } + if (query['@type'] === 'setVerbosity') { + this.td_functions.td_set_verbosity(query.verbosity); + return; + } + if (this.isPending) { + this.pendingQueries.push(query); + return; + } + query = this.prepareQuery(query); + this.td_functions.td_send(this.client, JSON.stringify(query)); + this.scheduleReceiveSoon(); + } + + receive() { + this.cancelReceive(); + if (this.wasFatalError) { + return; + } + try { + while (true) { + let msg = this.td_functions.td_receive(this.client); + if (!msg) { + break; + } + let response = this.prepareResponse(JSON.parse(msg)); + if ( + response['@type'] === 'updateAuthorizationState' && + response.authorization_state['@type'] === 'authorizationStateClosed' + ) { + this.close(response); + break; + } + this.callback(response); + } + + this.scheduleReceive(); + } catch (error) { + this.onFatalError(error); + } + } + + cancelReceive() { + if (this.receiveTimeout) { + clearTimeout(this.receiveTimeout); + delete this.receiveTimeout; + } + delete this.receiveSoon; + } + scheduleReceiveSoon() { + if (this.receiveSoon) { + return; + } + this.cancelReceive(); + this.receiveSoon = true; + this.scheduleReceiveIn(0.001); + } + scheduleReceive() { + if (this.receiveSoon) { + return; + } + this.cancelReceive(); + let timeout = this.td_functions.td_get_timeout(); + this.scheduleReceiveIn(timeout); + } + scheduleReceiveIn(timeout) { + //return; + log.debug('Scheduler receive in ' + timeout + 's'); + this.receiveTimeout = setTimeout(() => this.receive(), timeout * 1000); + } + + onFatalError(error) { + this.wasFatalError = true; + this.asyncOnFatalError(error); + } + + async close(last_update) { + // close db and cancell all timers + this.isClosing = true; + if (this.isStarted) { + log.debug('close worker: start'); + await this.tdfs.dbFileSystem.close(); + this.cancelReceive(); + log.debug('close worker: finish'); + } + this.callback(last_update); + } + + async asyncOnFatalError(error) { + await this.tdfs.dbFileSystem.sync(); + this.callback({ '@type': 'updateFatalError', error: error }); + } + + async saveFile(pid, file) { + let isSaving = this.savingFiles.has(pid); + this.savingFiles.set(pid, file); + if (isSaving) { + return; + } + let arr = await this.tdfs.inboundFileSystem.persist(pid, file.local.path); + file = this.savingFiles.get(pid); + file.idb_key = pid; + if (arr) { + file.arr = arr; + } + this.callback({ '@type': 'updateFile', file: file }, [arr.buffer]); + delete file.arr; + this.savingFiles.delete(pid); + } + + prepareFile(file) { + let pid = file.remote.id; + if (!pid) { + return file; + } + + if (file.local.is_downloading_active) { + this.tdfs.inboundFileSystem.forget(pid); + } else if (this.tdfs.inboundFileSystem.has(pid)) { + file.idb_key = pid; + return file; + } + + if (file.local.is_downloading_completed) { + this.saveFile(pid, file); + } + return file; + } + + prepareResponse(response) { + if (response['@type'] === 'file') { + return this.prepareFile(response); + } + for (var key in response) { + let field = response[key]; + if (field && typeof field === 'object') { + response[key] = this.prepareResponse(field); + } + } + return response; + } + + flushPendingQueries() { + this.isPending = false; + for (let query of this.pendingQueries) { + this.send(query); + } + } +} + +var client = new TdClient((e, t = []) => postMessage(e, t)); + +onmessage = function(e) { + try { + client.send(e.data); + } catch (error) { + client.onFatalError(error); + } +}; diff --git a/example/emscripten/tdweb/webpack.config.js b/example/emscripten/tdweb/webpack.config.js new file mode 100644 index 00000000..01817d44 --- /dev/null +++ b/example/emscripten/tdweb/webpack.config.js @@ -0,0 +1,79 @@ +const path = require('path'); +const CleanWebpackPlugin = require('clean-webpack-plugin'); +const UglifyJSPlugin = require('uglifyjs-webpack-plugin'); +const HtmlWebpackPlugin = require('html-webpack-plugin'); + +module.exports = { + entry: ['./src/index.js'], + output: { + filename: 'tdweb.js', + path: path.resolve(__dirname, 'dist'), + library: 'tdweb', + libraryTarget: 'umd', + umdNamedDefine: true + }, + devServer: { + contentBase: './dist' + }, + plugins: [ + new HtmlWebpackPlugin(), + new CleanWebpackPlugin(['dist'], {}) + //, new UglifyJSPlugin() + ], + module: { + rules: [ + { + test: /\.(js|jsx)$/, + exclude: /prebuilt/, + enforce: 'pre', + include: [path.resolve(__dirname, 'src')], + use: [ + { + loader: require.resolve('eslint-loader') + } + ] + }, + { + test: /worker\.(js|jsx)$/, + include: [path.resolve(__dirname, 'src')], + use: [ + { + loader: require.resolve('worker-loader') + } + ] + }, + { + test: /\.(js|jsx)$/, + exclude: /prebuilt/, + include: [path.resolve(__dirname, 'src')], + use: [ + { + loader: require.resolve('babel-loader') + } + ] + }, + { + test: /\.(wasm|mem)$/, + include: [path.resolve(__dirname, 'src')], + use: [ + { + loader: require.resolve('file-loader') + } + ] + } + ] + }, + node: { + dgram: 'empty', + fs: 'empty', + net: 'empty', + tls: 'empty', + crypto: 'empty', + child_process: 'empty' + }, + resolve: { + alias: { + ws$: 'fs' + } + } +}; diff --git a/td/generate/generate_common.cpp b/td/generate/generate_common.cpp index 8759c405..89114bee 100644 --- a/td/generate/generate_common.cpp +++ b/td/generate/generate_common.cpp @@ -36,7 +36,8 @@ int main() { {"\"td/tl/tl_object_parse.h\"", "\"td/tl/tl_object_store.h\""}, {"\"td/utils/buffer.h\""}); generate_cpp<>("auto/td/mtproto", "mtproto_api", "Slice", "Slice", - {"\"td/tl/tl_object_parse.h\"", "\"td/tl/tl_object_store.h\""}, {"\"td/utils/Slice.h\""}); + {"\"td/tl/tl_object_parse.h\"", "\"td/tl/tl_object_store.h\""}, + {"\"td/utils/Slice.h\"", "\"td/utils/UInt.h\""}); #ifdef TD_ENABLE_JNI generate_cpp(