Использование коннектора PXF HDFS для чтения и записи данных в формате SequenceFile между Greengage DB и HDFS

$ mkdir $PXF_BASE/servers/hadoop
$ cd $PXF_BASE/servers/hadoop

$ scp hdfsuser@namenode:/etc/hadoop/conf/core-site.xml .
$ scp hdfsuser@namenode:/etc/hadoop/conf/hdfs-site.xml .
$ scp hdfsuser@namenode:/etc/hadoop/conf/mapred-site.xml .
$ scp hdfsuser@namenode:/etc/hadoop/conf/yarn-site.xml .

$ pxf cluster sync

$ mkdir -p /tmp/pxf_seqfile_example/com/example/pxf/hdfs/writable/dataschema
$ cd /tmp/pxf_seqfile_example/com/example/pxf/hdfs/writable/dataschema
$ vi CustomerWritable.java

package com.example.pxf.hdfs.writable.dataschema;

import org.apache.hadoop.io.*;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.lang.reflect.Field;

public class CustomerWritable implements Writable {

    public int int1;
    public String st1, st2, st3;

    public CustomerWritable() {
        int1 = 0;
        st1 = "";
        st2 = "";
        st3 = "";
    }

    int getInt1() {
        return int1;
    }

    String getSt1() {
        return st1;
    }

    String getSt2() {
        return st2;
    }

    String getSt3() {
        return st3;
    }

    @Override
    public void write(DataOutput out) throws IOException {

        IntWritable intw = new IntWritable();
        intw.set(int1);
        intw.write(out);

        Text txt = new Text();
        txt.set(st1);
        txt.write(out);
        txt.set(st2);
        txt.write(out);
        txt.set(st3);
        txt.write(out);
    }

    @Override
    public void readFields(DataInput in) throws IOException {

        IntWritable intw = new IntWritable();
        intw.readFields(in);
        int1 = intw.get();

        Text txt = new Text();
        txt.readFields(in);
        st1 = txt.toString();
        txt.readFields(in);
        st2 = txt.toString();
        txt.readFields(in);
        st3 = txt.toString();
    }
}

$ javac -classpath /usr/lib/hadoop/hadoop-common.jar CustomerWritable.java
$ cd /tmp/pxf_seqfile_example/
$ jar cf customerwritable.jar com
$ cp customerwritable.jar /tmp/

$ scp /tmp/customerwritable.jar gpadmin@mdw:/home/gpadmin

$ cp /home/gpadmin/customerwritable.jar $PXF_BASE/lib/customerwritable.jar

$ pxf cluster sync
$ pxf cluster restart

CREATE WRITABLE EXTERNAL TABLE customers_w (
    id INT,
    name TEXT, 
    email TEXT, 
    address TEXT    
    )
    
    LOCATION ('pxf://tmp/pxf_examples/customers?PROFILE=hdfs:SequenceFile&DATA_SCHEMA=com.example.pxf.hdfs.writable.dataschema.CustomerWritable&SERVER=hadoop')
    FORMAT 'CUSTOM' (FORMATTER='pxfwritable_export');

INSERT INTO customers_w (id, name, email, address)
VALUES (1,'John Doe','john.doe@example.com','123 Elm Street'),
       (2,'Jane Smith','jane.smith@example.com','456 Oak Street'),
       (3,'Bob Brown','bob.brown@example.com','789 Pine Street'),
       (4,'Rob Stuart','rob.stuart@example.com','119 Willow Street');

$ hdfs dfs -ls /tmp/pxf_examples/customers

CREATE EXTERNAL TABLE customers_r (
    id INT,
    name TEXT,
    email TEXT,
    address TEXT
    )

    LOCATION ('pxf://tmp/pxf_examples/customers?PROFILE=hdfs:SequenceFile&DATA_SCHEMA=com.example.pxf.hdfs.writable.dataschema.CustomerWritable&SERVER=hadoop')
    FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import');

SELECT * FROM customers_r;

CREATE EXTERNAL TABLE customers_rkey (
    recordkey BIGINT,
    id INT,
    name TEXT,
    email TEXT,
    address TEXT
    )

    LOCATION ('pxf://tmp/pxf_examples/customers?PROFILE=hdfs:SequenceFile&DATA_SCHEMA=com.example.pxf.hdfs.writable.dataschema.CustomerWritable&SERVER=hadoop')
    FORMAT 'CUSTOM' (FORMATTER='pxfwritable_import');

SELECT * FROM customers_rkey;