Использование коннектора PXF HDFS для чтения и записи данных в формате ORC между Greengage DB и HDFS

$ mkdir $PXF_BASE/servers/hadoop
$ cd $PXF_BASE/servers/hadoop

$ scp hdfsuser@namenode:/etc/hadoop/conf/core-site.xml .
$ scp hdfsuser@namenode:/etc/hadoop/conf/hdfs-site.xml .
$ scp hdfsuser@namenode:/etc/hadoop/conf/mapred-site.xml .
$ scp hdfsuser@namenode:/etc/hadoop/conf/yarn-site.xml .

$ pxf cluster sync

{"id": 1,"name": "John Doe","ordered_items":["laptop", "monitor"]}
{"id": 2,"name": "Jane Smith","ordered_items":["keyboard", "mouse", "pad"]}
{"id": 3,"name": "Bob Brown","ordered_items":["headphones", "laptop"]}
{"id": 4,"name": "Alice Green","ordered_items":["webcam", "microphone", "mouse"]}

$ curl -O https://repo1.maven.org/maven2/org/apache/orc/orc-tools/2.2.2/orc-tools-2.2.2-uber.jar
$ java -jar /tmp/orc-tools-2.2.2-uber.jar convert /tmp/customers.json \
    --schema 'struct<id:int,name:string,ordered_items:array<string>>' \
    -o /tmp/customers.orc

$ hdfs dfs -mkdir -p /tmp/pxf_examples
$ hdfs dfs -put /tmp/customers.orc /tmp/pxf_examples/

CREATE EXTERNAL TABLE customers_r (
        id INT,
        name TEXT,
        ordered_items TEXT[]
    )
    
    LOCATION('pxf://tmp/pxf_examples/customers.orc?PROFILE=hdfs:orc&SERVER=hadoop')
    FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import');

SELECT * FROM customers_r;

SELECT * FROM customers_r WHERE ordered_items && '{"laptop", "monitor"}';

SELECT * FROM customers_r WHERE ordered_items[1] = 'headphones';

CREATE WRITABLE EXTERNAL TABLE customers_w(
        id INT,
        name TEXT, 
        ordered_items TEXT[]    
    )
    LOCATION ('pxf://tmp/pxf_examples/customers_w?PROFILE=hdfs:orc&SERVER=hadoop&COMPRESSION_CODEC=none')
    FORMAT 'CUSTOM' (FORMATTER='pxfwritable_export');

INSERT INTO customers_w 
VALUES (1, 'Bob Brown', ARRAY['monitor', 'webcam']),
       (2, 'John Doe', ARRAY['keyboard', 'microphone']),
       (3, 'Alice Green', ARRAY['headphones', 'pad']),
       (4, 'Jane Smith', ARRAY['laptop', 'mouse']);

$ hdfs dfs -get -f /tmp/pxf_examples/customers_w /tmp

$ java -jar /tmp/orc-tools-2.2.2-uber.jar meta -d /tmp/customers_w/39-0000000016_2.orc

Processing data file /tmp/customers_w/39-0000000016_2.orc [length: 675]
{"id":1,"name":"Bob Brown","ordered_items":["monitor","webcam"]}
{"id":2,"name":"John Doe","ordered_items":["keyboard","microphone"]}
{"id":3,"name":"Alice Green","ordered_items":["headphones","pad"]}
{"id":4,"name":"Jane Smith","ordered_items":["laptop","mouse"]}
______________________________________________________________________