下载编译好的二进制文件
https://github.com/wgzhao/Addax/releases/download/4.0.2/addax-4.0.2.tar.gz
tar -xzf addax-4.0.2.tar.gz cd addax-4.0.2
读http写pgsql配置文件
{ "job": { "setting": { "speed": { "channel": 1, "bytes": -1 } }, "content": [ { "reader": { "name": "httpreader", "parameter": { "connection": [ { "url": "http://127.0.0.1/dsideal_yy/admin/new_base/depinfo_getOrgTree" } ], "reqParams": { "org_id": ${org_id} }, "resultKey": "list", "method": "GET", "column": [ "id", "name" ], "username": "", "password": "", "headers": { "X-Powered-by": "Addax" } } }, "writer": { "name": "postgresqlwriter", "parameter": { "column": [ "id", "name" ], "preSql": [ "truncate table @table" ], "connection": [ { "jdbcUrl": "jdbc:postgresql://127.0.0.1:5432/edu_db", "table": [ "test" ] } ], "username": "root", "password": "123456", "writeMode": "insert" } } } ] } }
执行(-p加动态参数,配置文件中有${org_id}的参数,所以用-Dorg_id=400195方式赋值,多个就中间加空格,例:"-Dorg_id=400195 -Dperson_id=369587")
bin/addax.sh -p "-Dorg_id=400195" job/http2stream.json
读ES写pgsql配置文件
{ "job": { "setting": { "speed": { "byte": -1, "channel": 1 } }, "content": [ { "reader": { "name": "elasticsearchreader", "parameter": { "endpoint": "http://127.0.0.1:9200", "accessId": "", "accesskey": "", "index": "chat_record", "type": "_doc", "searchType": "dfs_query_then_fetch", "headers": {}, "scroll": "3m", "search": [ { "size": 2, "_source": [ "chat_type", "message_type", "receiver_user_name" ] } ], "column": [ "chat_type", "message_type", "receiver_user_name" ] } }, "writer": { "name": "postgresqlwriter", "parameter": { "column": [ "chat_type", "message_type", "receiver_user_name" ], "preSql": [ "truncate table @table" ], "connection": [ { "jdbcUrl": "jdbc:postgresql://127.0.0.1:5432/dsideal_db", "table": [ "test2" ] } ], "username": "root", "password": "123456", "writeMode": "insert" } }, "transformer": [ { "name": "dx_substr", //将第三个字段的内容,截取前两个字符 "parameter": { "columnIndex": 2, "paras": [ "0", "2" ] } } ] } ] } }