cascading.tap.Tap.resourceExists()方法的使用及代码示例

x33g5p2x  于2022-01-30 转载在 其他  
字(6.7k)|赞(0)|评价(0)|浏览(116)

本文整理了Java中cascading.tap.Tap.resourceExists()方法的一些代码示例,展示了Tap.resourceExists()的具体用法。这些代码示例主要来源于Github/Stackoverflow/Maven等平台,是从一些精选项目中提取出来的代码,具有较强的参考意义,能在一定程度帮忙到你。Tap.resourceExists()方法的具体详情如下:
包路径:cascading.tap.Tap
类名称:Tap
方法名:resourceExists

Tap.resourceExists介绍

[英]Method resourceExists returns true if the path represented by this instance exists.
[中]如果此实例表示的路径存在,则resourceExists方法返回true。

代码示例

代码示例来源:origin: elastic/elasticsearch-hadoop

@Override
public boolean resourceExists(Object conf) throws IOException {
  return actualTap.resourceExists(conf);
}

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( Config conf ) throws IOException
 {
 return original.resourceExists( conf );
 }

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( FlowProcess<? extends Config> flowProcess ) throws IOException
 {
 return original.resourceExists( flowProcess );
 }

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( Config conf ) throws IOException
 {
 return parent.resourceExists( conf );
 }

代码示例来源:origin: org.elasticsearch/elasticsearch-hadoop

@Override
public boolean resourceExists(Object conf) throws IOException {
  return actualTap.resourceExists(conf);
}

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( FlowProcess<? extends TConfig> flowProcess ) throws IOException
 {
 return original.resourceExists( processProvider.apply( flowProcess ) );
 }

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( TConfig conf ) throws IOException
 {
 return original.resourceExists( configProvider.apply( conf ) );
 }

代码示例来源:origin: cwensel/cascading

public boolean resourceExists( Config conf ) throws IOException
 {
 for( Tap tap : getTaps() )
  {
  if( !tap.resourceExists( conf ) )
   return false;
  }
 return true;
 }

代码示例来源:origin: cwensel/cascading

boolean allSourcesExist() throws IOException
 {
 for( Tap tap : sources.keySet() )
  {
  if( !tap.resourceExists( getConfig() ) )
   return false;
  }
 return true;
 }

代码示例来源:origin: cwensel/cascading

/**
 * Method resourceExists returns true if the path represented by this instance exists.
 *
 * @param flowProcess of type FlowProcess
 * @return true if the underlying resource already exists
 * @throws IOException when the status cannot be determined
 */
public boolean resourceExists( FlowProcess<? extends Config> flowProcess ) throws IOException
 {
 return resourceExists( flowProcess.getConfig() );
 }

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( Tap tap ) throws IOException
 {
 return tap.resourceExists( getConfig() );
 }

代码示例来源:origin: cwensel/cascading

@Override
public boolean resourceExists( Config conf ) throws IOException
 {
 for( Tap tap : getTaps() )
  {
  if( !tap.resourceExists( conf ) )
   return false;
  }
 return true;
 }

代码示例来源:origin: cwensel/cascading

public static long getSinkModified( Object config, Collection<Tap> sinks ) throws IOException
 {
 long sinkModified = Long.MAX_VALUE;
 for( Tap sink : sinks )
  {
  if( sink.isReplace() || sink.isUpdate() )
   sinkModified = -1L;
  else
   {
   if( !sink.resourceExists( config ) )
    sinkModified = 0L;
   else
    sinkModified = Math.min( sinkModified, sink.getModifiedTime( config ) ); // return youngest mod date
   }
  }
 return sinkModified;
 }

代码示例来源:origin: cwensel/cascading

public static long getSourceModified( Object confCopy, Iterator<Tap> values, long sinkModified ) throws IOException
 {
 long sourceModified = 0;
 while( values.hasNext() )
  {
  Tap source = values.next();
  if( source instanceof MultiSourceTap )
   return getSourceModified( confCopy, ( (MultiSourceTap) source ).getChildTaps(), sinkModified );
  sourceModified = source.getModifiedTime( confCopy );
  // source modified returns zero if does not exist
  // this should minimize number of times we touch any file meta-data server
  if( sourceModified == 0 && !source.resourceExists( confCopy ) )
   throw new FlowException( "source does not exist: " + source );
  if( sinkModified < sourceModified )
   return sourceModified;
  }
 return sourceModified;
 }

代码示例来源:origin: cascading/lingual-core

public String[] getChildIdentifiers( FileType<Config> fileType ) throws IOException
 {
 if( !( (Tap) fileType ).resourceExists( getSystemConfig() ) )
  throw new IllegalStateException( "resource does not exist: " + ( (Tap) fileType ).getFullIdentifier( getSystemConfig() ) );
 return fileType.getChildIdentifiers( getSystemConfig() );
 }

代码示例来源:origin: cascading/lingual-core

private boolean resourceExistsAndNotEmpty( SchemaDef schemaDef, Tap tap )
 {
 if( tap == null )
  return false;
 try
  {
  Object configCopy = platformBroker.getFlowProcess().getConfigCopy();
  if( !tap.resourceExists( configCopy ) )
   return false;
  if( !( tap instanceof FileType ) )
   return true;
  // is file not empty, or if directory, is any file not empty
  // just a safeguard against all files being empty
  return resourceNotEmpty( schemaDef, (FileType) tap, configCopy );
  }
 catch( IOException exception )
  {
  return false;
  }
 }

代码示例来源:origin: cwensel/cascading

/**
 * Method deleteSinksIfReplace deletes all sinks that are configured with the {@link cascading.tap.SinkMode#REPLACE} flag.
 *
 * @throws IOException
 */
public void deleteSinksIfReplace() throws IOException
 {
 // verify all sinks before incrementally deleting for a replace
 for( Tap tap : sinks.values() )
  {
  if( tap.isKeep() && tap.resourceExists( getConfig() ) )
   throw new FlowTapException( "resource exists and sink mode is KEEP, cannot overwrite: " + tap.getFullIdentifier( getFlowProcess() ) );
  }
 for( Tap tap : sinks.values() )
  {
  if( tap.isReplace() )
   deleteOrFail( tap );
  }
 }

代码示例来源:origin: cwensel/cascading

private void deleteOrFail( Tap tap ) throws IOException
 {
 if( !tap.resourceExists( getConfig() ) )
  return;
 if( !tap.deleteResource( getConfig() ) )
  throw new FlowTapException( "unable to delete resource: " + tap.getFullIdentifier( getFlowProcess() ) );
 }

代码示例来源:origin: cascading/lingual-core

private double getRowsInternal()
 {
 PlatformBroker platformBroker = getPlatformBroker();
 if( platformBroker == null )
  return super.getRows();
 SchemaCatalogManager catalog = platformBroker.getCatalogManager();
 if( catalog == null )
  return super.getRows();
 TableDef tableDef = getTapTable().getTableDef();
 try
  {
  Tap tap = catalog.createTapFor( tableDef, SinkMode.KEEP );
  if( tap != null )
   {
   if( !tap.resourceExists( platformBroker.getSystemConfig() ) )
    return 0.0;
   else if( tap instanceof FileType )
    return ( (FileType) tap ).getSize( platformBroker.getSystemConfig() ); // actually returns bytes
   }
  }
 catch( Exception exception )
  {
  LOG.warn( "unable to create tap for: " + tableDef );
  }
 return super.getRows();
 }
}

代码示例来源:origin: cwensel/cascading

@Test
public void testSkippedCascade() throws IOException
 {
 getPlatform().copyFromLocal( inputFileIps );
 String path = "skipped";
 Flow first = firstFlow( path + "/first", false );
 Flow second = secondFlow( first.getSink(), path + "/second" );
 Flow third = thirdFlow( second.getSink(), path + "/third" );
 Flow fourth = fourthFlow( third.getSink(), path + "/fourth" );
 CountingFlowListener flowListener = new CountingFlowListener();
 second.addListener( flowListener );
 Cascade cascade = new CascadeConnector( getProperties() ).connect( first, second, third, fourth );
 cascade.setFlowSkipStrategy( new FlowSkipStrategy()
  {
  public boolean skipFlow( Flow flow ) throws IOException
   {
   return true;
   }
  } );
 cascade.start();
 cascade.complete();
 assertEquals( 1, flowListener.skipped );
 assertFalse( "file exists", fourth.getSink().resourceExists( fourth.getConfig() ) );
 }

相关文章