code for compressed sort paper

iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/OrderSensitiveTimeDecoder.java

@@ -0,0 +1,106 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class OrderSensitiveTimeDecoder extends Decoder{
+
+    boolean isFirst = true;
+    long nowValue = 0;
+    int nowNum = 0;
+    int totalNum = 0;
+    int controlBitsOffset = 0;
+    int index = 0;
+    long[] pow = new long[] {1, 256, 65536, 16777216, 4294967296L, (long) Math.pow(256, 5), (long) Math.pow(256, 6), (long) Math.pow(256, 7)};
+    byte[] vals = new byte[4];
+    int[] map; // map index to length
+    boolean isLong = false;
+    public OrderSensitiveTimeDecoder() {
+        super(TSEncoding.ORDER_SENSITIVE_TIME);
+    }
+
+    @Override
+    public boolean hasNext(ByteBuffer buffer) throws IOException {
+        if (isFirst) {
+            if(isLong) {
+                map = new int[]{8, 1, 2, 4};
+            }
+            else{
+                map = new int[]{4, 0, 1, 2};
+            }
+            totalNum = readValueSize(buffer);
+            controlBitsOffset = buffer.limit()-4-(totalNum+3)/4;
+            return true;
+        }
+        if (nowNum < totalNum){
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public long readLong(ByteBuffer buffer) {
+        // forward decode
+        int valueLen;
+        if(isFirst) {
+            isFirst = false;
+            valueLen = readValueLen(buffer);
+            valueLen = 8;
+        } else {
+            valueLen = readValueLen(buffer);
+        }
+        nowValue += readForwardValueDelta(valueLen, buffer);
+        nowNum++;
+        return nowValue;
+    }
+
+    public long readForwardValueDelta(int byteNum, ByteBuffer buffer) {
+        long delta = 0;
+        for(int i=0; i<byteNum; i++){
+            long temp = buffer.get();
+            if(temp<0) temp = temp+256;
+            delta += temp*pow[i];
+        }
+        if(delta>Integer.MAX_VALUE && byteNum<=4){
+            delta = delta-Integer.MAX_VALUE-Integer.MAX_VALUE-2;
+        }
+        return delta;
+    }
+
+    private int readValueSize(ByteBuffer buffer) {
+        int size = 0;
+        int offset = buffer.limit() - 4;
+        for(int i=0; i<4; i++){
+            long temp = buffer.get(offset+i);
+            if(temp<0) temp = temp+256;
+            size += temp*pow[i];
+        }
+        return size;
+    }
+
+    public int readValueLen(ByteBuffer buffer){
+        if(index == 0){
+            byte temp = buffer.get(controlBitsOffset +nowNum/4);
+            for(int i=0; i<4; i++) {
+                vals[3-i] = (byte) (temp & 0x03);
+                temp >>= 2;
+            }
+        }
+        byte temp = vals[index];
+        index++;
+        if(index==4) index=0;
+        return map[temp];
+    }
+
+    @Override
+    public void reset() {
+        isFirst = true;
+        nowValue = 0;
+        nowNum = 0;
+        totalNum = 0;
+        controlBitsOffset = 0;
+    }
+}
+

iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/decoder/OrderSensitiveValueDecoder.java

@@ -0,0 +1,99 @@
+package org.apache.iotdb.tsfile.encoding.decoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class OrderSensitiveValueDecoder extends Decoder {
+    boolean isFirst = true;
+    int nowNum = 0;
+    int totalNum = 0;
+    int controlBitsOffset = 0;
+    int index = 0;
+    byte[] vals = new byte[4];
+    long[] pow = new long[] {1, 256, 65536, 16777216, 4294967296L, (long) Math.pow(256, 5), (long) Math.pow(256, 6), (long) Math.pow(256, 7)};
+    int[] map; // map index to length
+    boolean isLong = true;  // Determine the range of the data values
+
+    public OrderSensitiveValueDecoder() {
+        super(TSEncoding.ORDER_SENSITIVE_VALUE);
+    }
+
+    @Override
+    public boolean hasNext(ByteBuffer buffer) throws IOException {
+        if (isFirst) {
+            if(isLong) {
+                map = new int[]{8, 1, 2, 4};
+            }
+            else{
+                map = new int[]{4, 0, 1, 2};
+            }
+            totalNum = readValueSize(buffer);
+            controlBitsOffset = buffer.limit()-4-(totalNum+3)/4;
+            isFirst = false;
+            return true;
+        }
+        if (nowNum < totalNum){
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public long readLong(ByteBuffer buffer) {
+        int valueLen = readValueLen(buffer);
+        long value = readForwardValue(valueLen, buffer);
+        nowNum++;
+        return value;
+    }
+
+    @Override
+    public void reset() {
+        isFirst = true;
+        nowNum = 0;
+        totalNum = 0;
+        controlBitsOffset = 0;
+    }
+
+    private int readValueSize(ByteBuffer buffer) {
+        int size = 0;
+        int offset = buffer.limit() - 4;
+        for(int i=0; i<4; i++){
+            long temp = buffer.get(offset+i);
+            if(temp<0) temp = temp+256;
+            size += temp*pow[i];
+        }
+        return size;
+    }
+
+    public int readValueLen(ByteBuffer buffer){
+        if(index == 0){
+            byte temp = buffer.get(controlBitsOffset +nowNum/4);
+            for(int i=0; i<4; i++) {
+                vals[3-i] = (byte) (temp & 0x03);
+                temp >>= 2;
+            }
+        }
+        byte temp = vals[index];
+        index++;
+        if(index == 4) index = 0;
+        return map[temp];
+    }
+
+    public long readForwardValue(int byteNum, ByteBuffer buffer) {
+        long val = 0;
+        for(int i=0; i<byteNum; i++){
+            long temp = buffer.get();
+            if(temp<0) temp = temp+256;
+            val += temp*pow[i];
+        }
+        if(val>Integer.MAX_VALUE && byteNum<=4){
+            val = val-Integer.MAX_VALUE-Integer.MAX_VALUE-2;
+        }
+        if(val %2 == 0)
+            return val/2;
+        return -(val-1)/2;
+    }
+}
+

iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/encoding/encoder/OrderSensitiveTimeEncoder.java

@@ -0,0 +1,121 @@
+package org.apache.iotdb.tsfile.encoding.encoder;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+
+public class OrderSensitiveTimeEncoder extends Encoder{
+    byte[] lens;
+    byte[] vals;
+    int valNum = 0;
+    boolean isFirst = true;
+    long lastValue;
+    int index = 0;
+    boolean isLong = false; // Determine the range of the data values
+    byte[] map;
+    public OrderSensitiveTimeEncoder() {
+        super(TSEncoding.ORDER_SENSITIVE_TIME);
+    }
+
+    @Override
+    public void encode(long value, ByteArrayOutputStream out){
+        if (isFirst) {
+            if(isLong) {
+                map = new byte[]{-1,1,2,-1,3,-1,-1,-1,0};
+            } else {
+                map = new byte[]{1, 2, 3, -1, 0};
+            }
+            lens = new byte[1000];
+            vals = new byte[1000];
+            writeBits(value, out);
+            lastValue = value;
+            isFirst = false;
+        } else {
+            long delta = value-lastValue;
+            lastValue = value;
+            int byteNum = mapDeltaToLen(delta);
+            writeBits(delta, byteNum, out);
+        }
+        valNum++;
+    }
+
+    @Override
+    public void flush(ByteArrayOutputStream out) throws IOException {
+        // flush control bits
+        out.write(vals, 0, index);
+        out.write(lens,0, (valNum+3)/4);
+        // flush valNum
+        int byteNum = 4;
+        while(byteNum>0) {
+            out.write(valNum);
+            valNum = valNum>>8;
+            byteNum--;
+        }
+    }
+
+    public int mapDeltaToLen(long delta) {  //Provide options for lengths of 1, 2, 4, and 8
+        int newLen;
+        if(isLong) {
+            if (delta < 0) {
+                newLen = 8;
+            } else if (delta < 256) {
+                newLen = 1;
+            } else if (delta < 65536) {
+                newLen = 2;
+            } else if (delta < (long)Integer.MAX_VALUE*2) {
+                newLen = 4;
+            } else {
+                newLen = 8;
+            }
+        } else {
+            if (delta < 0) {
+                newLen = 4;
+            } else if (delta == 0) {
+                newLen = 0;
+            } else if (delta < 256) {
+                newLen = 1;
+            } else if (delta < 65536) {
+                newLen = 2;
+            } else {
+                newLen = 4;
+            }
+        }
+        return newLen;
+    }
+
+
+    private void writeBits(long value, int byteNum, ByteArrayOutputStream out){
+        // update control bits
+        if(valNum/4 >= lens.length-1){
+            byte[] expanded = new byte[lens.length*2];
+            System.arraycopy(lens, 0, expanded, 0, lens.length);
+            this.lens = expanded;
+        }
+        byte temp = map[byteNum];
+//        if(byteNum == 1) temp= 1;
+//        if(byteNum == 2) temp = 2;
+//        if(byteNum == 4) temp = 3;
+        lens[valNum/4] = (byte) (lens[valNum/4]|(temp<<(2*(3-valNum%4))));
+        // update data bits
+        while(byteNum>0) {
+            vals[index] = (byte) (value & 0xFFL);
+            value = value>>8;
+            byteNum--;
+            index++;
+            if(index == 1000) {
+                out.write(vals, 0, vals.length);
+                index = 0;
+            }
+        }
+    }
+
+    private void writeBits(long value, ByteArrayOutputStream out) {
+        int byteNum = 8;
+        while(byteNum>0) {
+            out.write((int)value);
+            value = value>>8;
+            byteNum--;
+        }
+    }
+}