Skip to content

Commit

Permalink
replace string device id with interface IDeviceID
Browse files Browse the repository at this point in the history
  • Loading branch information
shuwenwei committed Mar 21, 2024
1 parent 4c0aa7c commit 4620858
Show file tree
Hide file tree
Showing 39 changed files with 964 additions and 396 deletions.
25 changes: 25 additions & 0 deletions common/src/main/java/org/apache/tsfile/utils/Accountable.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.tsfile.utils;

public interface Accountable {
/** Return the memory usage of this object in bytes. Negative values are illegal. */
long ramBytesUsed();
}
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,9 @@ private static long sizeOfObject(Object o, int depth, long defSize) {
return 0;
}
long size;
if (o instanceof String) {
if (o instanceof Accountable) {
size = ((Accountable) o).ramBytesUsed();
} else if (o instanceof String) {
size = sizeOf((String) o);
} else if (o instanceof boolean[]) {
size = sizeOf((boolean[]) o);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.tsfile.file;

import java.io.IOException;
import java.io.OutputStream;

public interface IMetadataIndexEntry {

long getOffset();

void setOffset(long offset);

int serializeTo(OutputStream outputStream) throws IOException;

Comparable getCompareKey();

boolean isDeviceLevel();
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@

import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.file.MetaMarker;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.PlainDeviceID;
import org.apache.tsfile.read.reader.TsFileInput;
import org.apache.tsfile.utils.ReadWriteForEncodingUtils;
import org.apache.tsfile.utils.ReadWriteIOUtils;
Expand All @@ -33,7 +35,7 @@ public class ChunkGroupHeader {

private static final byte MARKER = MetaMarker.CHUNK_GROUP_HEADER;

private final String deviceID;
private final IDeviceID deviceID;

// this field does not need to be serialized.
private final int serializedSize;
Expand All @@ -43,7 +45,7 @@ public class ChunkGroupHeader {
*
* @param deviceID device ID
*/
public ChunkGroupHeader(String deviceID) {
public ChunkGroupHeader(IDeviceID deviceID) {
this.deviceID = deviceID;
this.serializedSize = getSerializedSize(deviceID);
}
Expand All @@ -52,8 +54,10 @@ public int getSerializedSize() {
return serializedSize;
}

private int getSerializedSize(String deviceID) {
int length = deviceID.getBytes(TSFileConfig.STRING_CHARSET).length;
private int getSerializedSize(IDeviceID deviceID) {
// TODO: add an interface in IDeviceID
int length =
((PlainDeviceID) deviceID).toStringID().getBytes(TSFileConfig.STRING_CHARSET).length;
return Byte.BYTES + ReadWriteForEncodingUtils.varIntSize(length) + length;
}

Expand All @@ -72,11 +76,12 @@ public static ChunkGroupHeader deserializeFrom(InputStream inputStream, boolean
}
}

// TODO: add an interface in IDeviceID
String deviceID = ReadWriteIOUtils.readVarIntString(inputStream);
if (deviceID == null || deviceID.isEmpty()) {
throw new IOException("DeviceId is empty");
}
return new ChunkGroupHeader(deviceID);
return new ChunkGroupHeader(new PlainDeviceID(deviceID));
}

/**
Expand All @@ -91,11 +96,12 @@ public static ChunkGroupHeader deserializeFrom(TsFileInput input, long offset, b
if (!markerRead) {
offsetVar++;
}
// TODO: add an interface in IDeviceID
String deviceID = input.readVarIntString(offsetVar);
return new ChunkGroupHeader(deviceID);
return new ChunkGroupHeader(new PlainDeviceID(deviceID));
}

public String getDeviceID() {
public IDeviceID getDeviceID() {
return deviceID;
}

Expand All @@ -109,15 +115,15 @@ public String getDeviceID() {
public int serializeTo(OutputStream outputStream) throws IOException {
int length = 0;
length += ReadWriteIOUtils.write(MARKER, outputStream);
length += ReadWriteIOUtils.writeVar(deviceID, outputStream);
length += ReadWriteIOUtils.writeVar(((PlainDeviceID) deviceID).toStringID(), outputStream);
return length;
}

@Override
public String toString() {
return "ChunkGroupHeader{"
+ "deviceID='"
+ deviceID
+ ((PlainDeviceID) deviceID).toStringID()
+ '\''
+ ", serializedSize="
+ serializedSize
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,16 @@
/** Only maintained when writing, not serialized to TsFile. */
public class ChunkGroupMetadata {

private String device;
private IDeviceID device;

private List<ChunkMetadata> chunkMetadataList;

public ChunkGroupMetadata(String device, List<ChunkMetadata> chunkMetadataList) {
public ChunkGroupMetadata(IDeviceID device, List<ChunkMetadata> chunkMetadataList) {
this.device = device;
this.chunkMetadataList = chunkMetadataList;
}

public String getDevice() {
public IDeviceID getDevice() {
return device;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.tsfile.file.metadata;

import org.apache.tsfile.file.IMetadataIndexEntry;
import org.apache.tsfile.utils.ReadWriteIOUtils;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;

public class DeviceMetadataIndexEntry implements IMetadataIndexEntry {
private IDeviceID deviceID;
private long offset;

public DeviceMetadataIndexEntry(IDeviceID deviceID, long offset) {
this.deviceID = deviceID;
this.offset = offset;
}

public IDeviceID getDeviceID() {
return deviceID;
}

@Override
public long getOffset() {
return offset;
}

public void setDeviceID(IDeviceID deviceID) {
this.deviceID = deviceID;
}

@Override
public void setOffset(long offset) {
this.offset = offset;
}

@Override
public int serializeTo(OutputStream outputStream) throws IOException {
int byteLen = 0;
byteLen += deviceID.serialize(outputStream);
byteLen += ReadWriteIOUtils.write(offset, outputStream);
return byteLen;
}

@Override
public Comparable getCompareKey() {
return deviceID;
}

@Override
public boolean isDeviceLevel() {
return true;
}

public static DeviceMetadataIndexEntry deserializeFrom(ByteBuffer buffer) {
IDeviceID device = IDeviceID.deserializeFrom(buffer);
long offset = ReadWriteIOUtils.readLong(buffer);
return new DeviceMetadataIndexEntry(device, offset);
}

public static DeviceMetadataIndexEntry deserializeFrom(InputStream inputStream)
throws IOException {
IDeviceID device = IDeviceID.deserializeFrom(inputStream);
long offset = ReadWriteIOUtils.readLong(inputStream);
return new DeviceMetadataIndexEntry(device, offset);
}

@Override
public String toString() {
return "<" + deviceID + "," + offset + ">";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.tsfile.file.metadata;

import org.apache.tsfile.utils.Accountable;
import org.apache.tsfile.utils.ReadWriteIOUtils;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;

/** Device id interface. */
public interface IDeviceID extends Comparable<IDeviceID>, Accountable {

int serialize(ByteBuffer byteBuffer);

int serialize(OutputStream outputStream) throws IOException;

byte[] getBytes();

boolean isEmpty();

static IDeviceID deserializeFrom(ByteBuffer byteBuffer) {
return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(byteBuffer));
}

static IDeviceID deserializeFrom(InputStream inputStream) throws IOException {
return new PlainDeviceID(ReadWriteIOUtils.readVarIntString(inputStream));
}
}
Loading

0 comments on commit 4620858

Please sign in to comment.