dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

No more protobuf in JS executor

Browse Source
This commit is contained in:
Andrew Shvayka 2019-12-03 12:38:52 +02:00
parent 8569b7dac2
commit 5fe40b70b3
8 changed files with 25 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
View File

@ -101,22 +101,6 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
requestBuilder.clientId("producer-js-invoke-" + nodeIdProvider.getNodeId());
requestBuilder.defaultTopic(requestTopic);
requestBuilder.encoder(new RemoteJsRequestEncoder());
requestBuilder.enricher((request, responseTopic, requestId) -> {
JsInvokeProtos.RemoteJsRequest.Builder remoteRequest = JsInvokeProtos.RemoteJsRequest.newBuilder();
if (request.hasCompileRequest()) {
remoteRequest.setCompileRequest(request.getCompileRequest());
}
if (request.hasInvokeRequest()) {
remoteRequest.setInvokeRequest(request.getInvokeRequest());
}
if (request.hasReleaseRequest()) {
remoteRequest.setReleaseRequest(request.getReleaseRequest());
}
remoteRequest.setResponseTopic(responseTopic);
remoteRequest.setRequestIdMSB(requestId.getMostSignificantBits());
remoteRequest.setRequestIdLSB(requestId.getLeastSignificantBits());
return remoteRequest.build();
});
TBKafkaConsumerTemplate.TBKafkaConsumerTemplateBuilder<JsInvokeProtos.RemoteJsResponse> responseBuilder = TBKafkaConsumerTemplate.builder();
responseBuilder.settings(kafkaSettings);

10
application/src/main/java/org/thingsboard/server/service/script/RemoteJsRequestEncoder.java
View File

@ -15,15 +15,23 @@
*/
package org.thingsboard.server.service.script;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.kafka.TbKafkaEncoder;
import java.nio.charset.StandardCharsets;
/**
* Created by ashvayka on 25.09.18.
*/
public class RemoteJsRequestEncoder implements TbKafkaEncoder<JsInvokeProtos.RemoteJsRequest> {
@Override
public byte[] encode(JsInvokeProtos.RemoteJsRequest value) {
return value.toByteArray();
try {
return JsonFormat.printer().print(value).getBytes(StandardCharsets.UTF_8);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
}
}

6
application/src/main/java/org/thingsboard/server/service/script/RemoteJsResponseDecoder.java
View File

@ -15,10 +15,12 @@
*/
package org.thingsboard.server.service.script;
import com.google.protobuf.util.JsonFormat;
import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.kafka.TbKafkaDecoder;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
* Created by ashvayka on 25.09.18.
@ -27,6 +29,8 @@ public class RemoteJsResponseDecoder implements TbKafkaDecoder<JsInvokeProtos.Re
@Override
public JsInvokeProtos.RemoteJsResponse decode(byte[] data) throws IOException {
return JsInvokeProtos.RemoteJsResponse.parseFrom(data);
JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder();
JsonFormat.parser().ignoringUnknownFields().merge(new String(data, StandardCharsets.UTF_8), builder);
return builder.build();
}
}

9
application/src/main/proto/jsinvoke.proto
View File

@ -26,12 +26,9 @@ enum JsInvokeErrorCode {
}
message RemoteJsRequest {
string responseTopic = 1;
int64 requestIdMSB = 2;
int64 requestIdLSB = 3;
JsCompileRequest compileRequest = 4;
JsInvokeRequest invokeRequest = 5;
JsReleaseRequest releaseRequest = 6;
JsCompileRequest compileRequest = 1;
JsInvokeRequest invokeRequest = 2;
JsReleaseRequest releaseRequest = 3;
}
message RemoteJsResponse {

14
common/queue/src/main/java/org/thingsboard/server/kafka/TBKafkaProducerTemplate.java
View File

@ -49,9 +49,6 @@ public class TBKafkaProducerTemplate<T> {
private final KafkaProducer<String, byte[]> producer;
private final TbKafkaEncoder<T> encoder;
@Builder.Default
private TbKafkaEnricher<T> enricher = ((value, responseTopic, requestId) -> value);
private final TbKafkaPartitioner<T> partitioner;
private ConcurrentMap<String, List<PartitionInfo>> partitionInfoMap;
@Getter
@ -61,7 +58,7 @@ public class TBKafkaProducerTemplate<T> {
private final TbKafkaSettings settings;
@Builder
private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder<T> encoder, TbKafkaEnricher<T> enricher,
private TBKafkaProducerTemplate(TbKafkaSettings settings, TbKafkaEncoder<T> encoder,
TbKafkaPartitioner<T> partitioner, String defaultTopic, String clientId) {
Properties props = settings.toProps();
props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
@ -72,7 +69,6 @@ public class TBKafkaProducerTemplate<T> {
this.settings = settings;
this.producer = new KafkaProducer<>(props);
this.encoder = encoder;
this.enricher = enricher;
this.partitioner = partitioner;
this.defaultTopic = defaultTopic;
}
@ -93,14 +89,6 @@ public class TBKafkaProducerTemplate<T> {
}
}
T enrich(T value, String responseTopic, UUID requestId) {
if (enricher != null) {
return enricher.enrich(value, responseTopic, requestId);
} else {
return value;
}
}
public Future<RecordMetadata> send(String key, T value, Callback callback) {
return send(key, value, null, callback);
}

24
common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaEnricher.java
View File

@ -1,24 +0,0 @@
/**
* Copyright © 2016-2019 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.kafka;
import java.util.UUID;
public interface TbKafkaEnricher<T> {
T enrich(T value, String responseTopic, UUID requestId);
}

11
common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaRequestTemplate.java
View File

@ -144,11 +144,11 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
tickSize = pendingRequests.size();
if (nextCleanupMs < tickTs) {
//cleanup;
pendingRequests.entrySet().forEach(kv -> {
if (kv.getValue().expTime < tickTs) {
ResponseMetaData<Response> staleRequest = pendingRequests.remove(kv.getKey());
pendingRequests.forEach((key, value) -> {
if (value.expTime < tickTs) {
ResponseMetaData<Response> staleRequest = pendingRequests.remove(key);
if (staleRequest != null) {
log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", kv.getKey(), staleRequest.expTime, tickTs);
log.trace("[{}] Request timeout detected, expTime [{}], tickTs [{}]", key, staleRequest.expTime, tickTs);
staleRequest.future.setException(new TimeoutException());
}
}
@ -189,13 +189,12 @@ public class TbKafkaRequestTemplate<Request, Response> extends AbstractTbKafkaTe
SettableFuture<Response> future = SettableFuture.create();
ResponseMetaData<Response> responseMetaData = new ResponseMetaData<>(tickTs + maxRequestTimeout, future);
pendingRequests.putIfAbsent(requestId, responseMetaData);
request = requestTemplate.enrich(request, responseTemplate.getTopic(), requestId);
log.trace("[{}] Sending request, key [{}], expTime [{}]", requestId, key, responseMetaData.expTime);
requestTemplate.send(key, request, headers, (metadata, exception) -> {
if (exception != null) {
log.trace("[{}] Failed to post the request", requestId, exception);
} else {
log.trace("[{}] Posted the request", requestId, metadata);
log.trace("[{}] Posted the request: {}", requestId, metadata);
}
});
return future;

5
common/queue/src/main/java/org/thingsboard/server/kafka/TbKafkaSettings.java
View File

@ -32,9 +32,8 @@ import java.util.Properties;
@Component
public class TbKafkaSettings {
public static final String REQUEST_ID_HEADER = "requestId";
public static final String RESPONSE_TOPIC_HEADER = "responseTopic";
static final String REQUEST_ID_HEADER = "requestId";
static final String RESPONSE_TOPIC_HEADER = "responseTopic";
@Value("${kafka.bootstrap.servers}")
private String servers;