com.addthis.hydra.data.query.Query类的使用及代码示例

x33g5p2x  于2022-01-29 转载在 其他  
字(12.3k)|赞(0)|评价(0)|浏览(257)

本文整理了Java中com.addthis.hydra.data.query.Query类的一些代码示例,展示了Query类的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Query类的具体详情如下:
包路径:com.addthis.hydra.data.query.Query
类名称:Query

Query介绍

[英]Object representation of a tree query.
[中]树查询的对象表示。

代码示例

代码示例来源:origin: addthis/hydra

  1. @Test
  2. public void pipeline() {
  3. String path = "+:+hits,+nodes$+foo=123/+/++123/+%top=hit/a,b,c/|foo/|+bar/*/+%goo/(1-5)+";
  4. String[] ops = {"sort"};
  5. Query q = new Query("job", new String[] { path }, ops);
  6. Query subQ = q.createPipelinedQuery();
  7. System.out.println(subQ.toString());
  8. }
  9. }

代码示例来源:origin: addthis/hydra

  1. public QueryEntryInfo getStat() {
  2. QueryEntryInfo stat = new QueryEntryInfo();
  3. stat.paths = query.getPaths();
  4. stat.uuid = query.queryId();
  5. stat.ops = opsLog;
  6. stat.job = query.getJob();
  7. stat.sources = query.getParameter("sources");
  8. stat.remoteip = query.getParameter("remoteip");
  9. stat.sender = query.getParameter("sender");
  10. stat.lines = preOpLines.get();
  11. stat.sentLines = postOpLines.get();
  12. stat.runTime = getRunTime();
  13. stat.startTime = startTime;
  14. stat.tasks = lastSourceInfo;
  15. stat.state = queryState;
  16. return stat;
  17. }

代码示例来源:origin: addthis/hydra

  1. void checkForStragglersMeans() {
  2. int totalTasks = sourceAggregator.totalTasks;
  3. int numRemaining = totalTasks - sourceAggregator.completed;
  4. int tasksDoneCutoff = Math.max(1, (int) Math.ceil(AggregateConfig.stragglerCheckHostPercentage * totalTasks));
  5. long elapsedTime = JitterClock.globalTime() - sourceAggregator.startTime;
  6. double timeCutoff = AggregateConfig.stragglerCheckMeanRuntimeFactor * getMeanRuntime();
  7. if (numRemaining == 0) {
  8. if (MeshSourceAggregator.log.isDebugEnabled() || sourceAggregator.query.isTraced()) {
  9. Query.traceLog.info("Straggler checker for {} detected query done. Exiting.",
  10. sourceAggregator.query.uuid());
  11. }
  12. } else if ((numRemaining <= tasksDoneCutoff) && (elapsedTime > timeCutoff)) {
  13. handleStragglers();
  14. }
  15. }

代码示例来源:origin: addthis/hydra

  1. /**
  2. * @return first a query suitable for the next query worker in the stack
  3. */
  4. public Query createPipelinedQuery() {
  5. Query newQuery = cloneTo(new Query());
  6. if (ops != null && ops.length > 0) {
  7. String[] newops = new String[ops.length - 1];
  8. System.arraycopy(ops, 1, newops, 0, newops.length);
  9. newQuery.ops = newops;
  10. String pop = ops[0];
  11. ops = new String[]{pop};
  12. }
  13. return newQuery;
  14. }

代码示例来源:origin: addthis/hydra

  1. Query query = new Query(job, new String[]{path}, new String[]{kv.getValue("ops"), kv.getValue("rops")});
  2. query.setTraced(kv.getIntValue("trace", 0) == 1);
  3. query.setParameterIfNotYetSet("hosts", kv.getValue("hosts"));
  4. query.setParameterIfNotYetSet("gate", kv.getValue("gate"));
  5. query.setParameterIfNotYetSet("originalrequest", kv.getValue("originalrequest"));
  6. SocketAddress remoteIP = ctx.channel().remoteAddress();
  7. if (remoteIP instanceof InetSocketAddress) { // only log implementations with known methods
  8. query.setParameterIfNotYetSet("remoteip", ((InetSocketAddress) remoteIP).getAddress().getHostAddress());
  9. query.setParameterIfNotYetSet("allocator", kv.getValue("allocator"));
  10. query.setParameterIfNotYetSet("allowPartial", kv.getValue("allowPartial"));
  11. query.setParameterIfNotYetSet("tasks", kv.getValue("tasks"));
  12. query.setParameter("allocator", "lazy");
  13. query.setParameter("maxSimul", maxSimul);
  14. String gdriveAccessToken = kv.getValue("accesstoken");
  15. int timeout = Math.min(kv.getIntValue("timeout", maxQueryTime), maxQueryTime);
  16. query.setParameterIfNotYetSet("timeout", timeout);
  17. query.setParameter("sender", kv.getValue("sender"));
  18. query.setParameter("injectSource", kv.getValue("injectSource"));
  19. .put("query.path", query.getPaths()[0])
  20. .put("query.hosts", query.getParameter("hosts"))
  21. .put("query.ops", query.getOps())
  22. .put("trace", query.isTraced())
  23. .put("sources", query.getParameter("sources"))
  24. .put("time", System.currentTimeMillis())
  25. .put("job.id", query.getJob())

代码示例来源:origin: addthis/hydra

  1. @Test
  2. public void testCompact() {
  3. String path = "+:+hits,+nodes$+foo=123/+/++123/+%top=hit/a,b,c/|foo/|+bar/*/+%goo/(1-5)+";
  4. Query q = new Query("job", new String[] { path }, null);
  5. Assert.assertEquals(path, q.getPathString(q.getQueryPaths().get(0)));
  6. }

代码示例来源:origin: addthis/hydra

  1. Query query = new Query(job, paths.toArray(new String[paths.size()]), ops.toArray(new String[ops.size()]));
  2. query.setTraced(traced);
  3. for (Entry<String, String> e : qparam.entrySet()) {
  4. query.setParameter(e.getKey(), e.getValue());

代码示例来源:origin: addthis/hydra

  1. protected void writeQuery(ChannelHandlerContext ctx, Query query, ChannelPromise promise) throws Exception {
  2. String[] opsLog = query.getOps();
  3. Query remoteQuery = query.createPipelinedQuery();
  4. boolean allowPartial = Boolean.valueOf(query.getParameter("allowPartial"));
  5. Set<Integer> tasks = parseTasks(query.getParameter("tasks"));
  6. List<QueryTaskSource[]> sourcesPerDir = new ArrayList<>(2);
  7. for (String combinedUnresolved : JOB_SPLITTER.split(query.getJob())) {
  8. String jobIdOrAlias = getJobWithoutSubdirectory(combinedUnresolved);
  9. String subdirectory = getJobSubdirectory(combinedUnresolved);

代码示例来源:origin: addthis/hydra

  1. Query q = new Query(null,
  2. new String[]{kv.getValue("query", kv.getValue("path", ""))},
  3. null);
  4. String qo = "{path:" + kv.getValue("query", kv.getValue("path", "")) + "}";
  5. Query q = CodecJSON.decodeString(Query.class, qo);
  6. writer.write(q.getPaths()[0]);
  7. break;

代码示例来源:origin: addthis/hydra

  1. static void sendTimeout(QueryEntry entry, long timeout) {
  2. String message = "[timeout.watcher] timeout: " + timeout +
  3. " has been exceeded, canceling query: " + entry.query.uuid();
  4. if (!entry.tryFailure(new TimeoutException(message))) {
  5. log.warn("QUERY TIMEOUT FAILURE: query: {}", entry.query.uuid());
  6. }
  7. }
  8. }

代码示例来源:origin: addthis/hydra

  1. protected void writeQuery(final ChannelHandlerContext ctx, Query msg, ChannelPromise promise)
  2. throws Exception {
  3. this.requestPromise = promise;
  4. this.queryUser = new DataChannelOutputToNettyBridge(ctx, promise);
  5. this.query = msg;
  6. query.queryPromise = queryPromise;
  7. // create a processor chain based in query ops terminating the query user
  8. this.opProcessorConsumer = query.newProcessor(queryUser, opPromise);
  9. queryEntry = new QueryEntry(query, opsLog, this, aggregator);
  10. // Check if the uuid is repeated, then make a new one
  11. if (queryTracker.running.putIfAbsent(query.uuid(), queryEntry) != null) {
  12. throw new QueryException("Query uuid somehow already in use : " + query.uuid());
  13. }
  14. log.debug("Executing.... {} {}", query.uuid(), queryEntry);
  15. ctx.pipeline().remove(this);
  16. opPromise.addListener(this);
  17. queryPromise.addListener(this);
  18. requestPromise.addListener(this);
  19. ctx.write(opProcessorConsumer, queryPromise);
  20. }

代码示例来源:origin: addthis/hydra

  1. public String getParameter(String key) {
  2. return getParameter(key, null);
  3. }

代码示例来源:origin: addthis/hydra

  1. @Before
  2. public void setup() throws Exception {
  3. query = new Query("jobid", new String[]{"path"}, new String[]{"ops"});
  4. MockitoAnnotations.initMocks(this);
  5. QueryTaskSource[] taskSources = new QueryTaskSource[]{taskSource0, taskSource1, taskSource2, taskSource3};
  6. for (QueryTaskSource x: taskSources) {
  7. stubSelectedSource(x);
  8. }
  9. MeshSourceAggregator underlying = new MeshSourceAggregator(taskSources, null, null, query);
  10. underlying.queryPromise = mock(ChannelProgressivePromise.class);
  11. underlying.consumer = mock(DataChannelOutput.class);
  12. underlying.channelWritable = true;
  13. underlying.executor = executor;
  14. sourceAggregator = spy(underlying);
  15. queryTask = new QueryTask(sourceAggregator);
  16. }

代码示例来源:origin: addthis/hydra

  1. public Query setParameterIfNotYetSet(String key, Object value) {
  2. if (params.get(key) == null) {
  3. setParameter(key, value);
  4. }
  5. return this;
  6. }

代码示例来源:origin: addthis/hydra

  1. /**
  2. * Part 1 - SETUP
  3. * Initialize query run -- parse options, create Query object
  4. */
  5. protected void setup() throws Exception {
  6. long startTime = System.currentTimeMillis();
  7. MeshQuerySource.queueTimes.update(creationTime - startTime, TimeUnit.MILLISECONDS);
  8. query = CodecJSON.decodeString(Query.class, options.get("query"));
  9. // set as soon as possible (and especially before creating op processor)
  10. query.queryPromise = bridge.queryPromise;
  11. // Parse the query and return a reference to the last QueryOpProcessor.
  12. ChannelProgressivePromise opPromise =
  13. new DefaultChannelProgressivePromise(null, ImmediateEventExecutor.INSTANCE);
  14. queryOpProcessor = query.newProcessor(bridge, opPromise);
  15. }

代码示例来源:origin: addthis/hydra

  1. /**
  2. * Performs a query search, writes the results to a data channel. This function does not break the execution of the
  3. * query if the client channel gets closed.
  4. *
  5. * @param query A Query object that contains the path or paths of the root query.
  6. * @param result A DataChannelOutput to which the result will be written. In practice, this will be the head of
  7. * a QueryOpProcessor that represents the first operator in a query, which in turn sends its output
  8. * to another QueryOpProcessor and the last will send its output to a DataChannelOutput sending bytes
  9. * back to meshy, usually defined at the MQSource side of code.
  10. * @param queryPromise A wrapper for a boolean flag that gets set to true by MQSource in case the user
  11. * cancels the query at the MQMaster side.
  12. */
  13. public void search(Query query, DataChannelOutput result,
  14. ChannelProgressivePromise queryPromise) throws QueryException {
  15. for (QueryElement[] path : query.getQueryPaths()) {
  16. if (!(queryPromise.isDone())) {
  17. search(path, result, queryPromise);
  18. }
  19. }
  20. }

代码示例来源:origin: addthis/hydra

  1. @Override
  2. public void cancel(String message) {
  3. log.warn(query.uuid() + " cancel called on handle " + consumer + " message: " + message);
  4. if (engine != null) {
  5. interrupt();
  6. }
  7. }
  8. }

代码示例来源:origin: addthis/hydra

  1. QueryEntry(Query query, String[] opsLog, TrackerHandler trackerHandler, MeshSourceAggregator aggregator) {
  2. this.query = query;
  3. this.opsLog = opsLog;
  4. this.trackerHandler = trackerHandler;
  5. this.preOpLines = new AtomicInteger();
  6. this.postOpLines = new AtomicInteger();
  7. this.aggregator = aggregator;
  8. final String timeoutInSeconds = query.getParameter("timeout");
  9. this.startTime = System.currentTimeMillis();
  10. if (timeoutInSeconds != null) {
  11. waitTime = Integer.parseInt(timeoutInSeconds);
  12. } else {
  13. waitTime = -1;
  14. }
  15. }

代码示例来源:origin: addthis/hydra

  1. /**
  2. * Part 2 - ENGINE CACHE
  3. * Get a QueryEngine for our query -- check the cache for a suitable candidate, otherwise make one.
  4. * Most of this logic is handled by the QueryEngineCache.get() function.
  5. */
  6. protected QueryEngine getEngine() throws Exception {
  7. final long engineGetStartTime = System.currentTimeMillis();
  8. // Use the canonical path stored in the canonicalDirString to create a QueryEngine. By that way
  9. // if the alias changes new queries will use the latest available
  10. // database and the old engines will be automatically closed after their TTL expires.
  11. QueryEngine engine = MeshQuerySource.queryEngineCache.getAndLease(goldDirString);
  12. final long engineGetDuration = System.currentTimeMillis() - engineGetStartTime;
  13. MeshQuerySource.engineGetTimer.update(engineGetDuration, TimeUnit.MILLISECONDS);
  14. if (engine == null) //Cache returned null -- this doesn't mean cache miss. It means something went fairly wrong
  15. {
  16. log.warn("[QueryReference] Unable to retrieve queryEngine for query: {}, key: {} after waiting: {}ms",
  17. query.uuid(), goldDirString, engineGetDuration);
  18. throw new DataChannelError("Unable to retrieve queryEngine for query: " + query.uuid() +
  19. ", key: " + goldDirString + " after waiting: " + engineGetDuration + "ms");
  20. } //else we got an engine so we're good -- maybe this logic should be in the cache get
  21. if ((engineGetDuration > MeshQuerySource.slowQueryThreshold) || log.isDebugEnabled() || query.isTraced()) {
  22. Query.traceLog.info(
  23. "[QueryReference] Retrieved queryEngine for query: {}, key:{} after waiting: {}ms. slow={}",
  24. query.uuid(), goldDirString, engineGetDuration,
  25. engineGetDuration > MeshQuerySource.slowQueryThreshold);
  26. }
  27. return engine;
  28. }

代码示例来源:origin: addthis/hydra

  1. @Override
  2. public void run() {
  3. try {
  4. long currentTime = System.currentTimeMillis();
  5. for (QueryEntry queryEntry : running.values()) {
  6. if (queryEntry.waitTime <= 0) {
  7. continue;
  8. }
  9. long queryDuration = currentTime - queryEntry.startTime;
  10. // wait time is in seconds
  11. double queryDurationInSeconds = queryDuration / 1000.0;
  12. if (queryDurationInSeconds < queryEntry.waitTime) {
  13. log.info("query: {} running for: {} timeout is: {}",
  14. queryEntry.query.uuid(), queryDurationInSeconds, queryEntry.waitTime);
  15. } else {
  16. log.warn("QUERY TIMEOUT query: {} running for: {} timeout is: {}",
  17. queryEntry.query.uuid(), queryDurationInSeconds, queryEntry.waitTime);
  18. // sanity check duration
  19. if (queryDurationInSeconds > (2 * queryEntry.waitTime)) {
  20. log.warn("query: {} query duration was insane, resetting to waitTime for logging. startTime: {}",
  21. queryEntry.query.uuid(), queryEntry.startTime);
  22. }
  23. sendTimeout(queryEntry, queryEntry.waitTime);
  24. }
  25. }
  26. } catch (Throwable ex) {
  27. log.error("error while running timeout watcher; swallowing to keep thread alive", ex);
  28. }
  29. }

相关文章