public class ColumnExplorer extends Object
Modifier and Type | Class and Description |
---|---|
static interface |
ColumnExplorer.ImplicitFileColumn |
static class |
ColumnExplorer.ImplicitFileColumns
Columns that give information from where file data comes from.
|
static class |
ColumnExplorer.ImplicitInternalFileColumns
Columns that give internal information about file or its parts.
|
Constructor and Description |
---|
ColumnExplorer(OptionManager optionManager)
Constructor for using the column explorer to probe existing columns in the
ProjectRecordBatch . |
ColumnExplorer(OptionManager optionManager,
List<SchemaPath> columns)
Helper class that encapsulates logic for sorting out columns
between actual table columns, partition columns and implicit file columns.
|
Modifier and Type | Method and Description |
---|---|
boolean |
containsImplicitColumns()
Checks if current column selection contains implicit columns.
|
boolean |
containsPartitionColumns()
Checks if current column selection contains partition columns.
|
static List<String> |
getImplicitColumnsNames(SchemaConfig schemaConfig)
Returns list with implicit column names taken from specified
SchemaConfig . |
static String |
getImplicitColumnValue(ColumnExplorer.ImplicitFileColumn column,
org.apache.hadoop.fs.Path filePath,
org.apache.hadoop.fs.FileSystem fs)
Returns implicit column value for specified implicit file column.
|
static String |
getImplicitColumnValue(ColumnExplorer.ImplicitFileColumn column,
org.apache.hadoop.fs.Path filePath,
org.apache.hadoop.fs.FileSystem fs,
Integer index,
Long start,
Long length)
Returns implicit column value for specified implicit file column.
|
static List<ColumnExplorer.ImplicitFileColumn> |
getImplicitFileColumns()
Returns list of implicit file columns which includes all elements from
ColumnExplorer.ImplicitFileColumns ,
ColumnExplorer.ImplicitInternalFileColumns.LAST_MODIFIED_TIME and
ColumnExplorer.ImplicitInternalFileColumns.USE_METADATA columns. |
static List<String> |
getPartitionColumnNames(FileSelection selection,
ColumnNamesOptions columnNamesOptions)
Returns list with partition column names.
|
static int |
getPartitionDepth(FileSelection selection) |
List<SchemaPath> |
getTableColumns() |
static Map<String,ColumnExplorer.ImplicitFileColumns> |
initImplicitFileColumns(OptionManager optionManager)
Creates case insensitive map with implicit file columns as keys and
appropriate ImplicitFileColumns enum as values
|
static Map<String,ColumnExplorer.ImplicitInternalFileColumns> |
initImplicitInternalFileColumns(OptionManager optionManager)
Creates case insensitive map with implicit internal file columns as keys and
appropriate ImplicitFileColumns enum as values
|
boolean |
isImplicitOrInternalFileColumn(String name)
Checks whether given column is implicit or internal.
|
static boolean |
isPartitionColumn(OptionManager optionManager,
SchemaPath column)
Checks if given column is partition or not.
|
static boolean |
isPartitionColumn(String partitionDesignator,
String path)
Checks if given column is partition or not.
|
boolean |
isStarQuery() |
static List<String> |
listPartitionValues(org.apache.hadoop.fs.Path file,
org.apache.hadoop.fs.Path root,
boolean hasDirsOnly)
Compares root and file path to determine directories
that are present in the file path but absent in root.
|
static String[] |
parsePartitions(org.apache.hadoop.fs.Path file,
org.apache.hadoop.fs.Path root,
boolean hasDirsOnly)
Low-level parse of partitions, returned as a string array.
|
Map<String,String> |
populateColumns(org.apache.hadoop.fs.Path filePath,
List<String> partitionValues,
boolean includeFileImplicitColumns,
org.apache.hadoop.fs.FileSystem fs)
Creates map with implicit and internal columns where key is column name, value is columns actual value.
|
Map<String,String> |
populateColumns(org.apache.hadoop.fs.Path filePath,
List<String> partitionValues,
boolean includeFileImplicitColumns,
org.apache.hadoop.fs.FileSystem fs,
int index,
long start,
long length)
Creates map with implicit and internal columns where key is column name, value is columns actual value.
|
Map<String,String> |
populateImplicitColumns(org.apache.hadoop.fs.Path filePath,
List<String> partitionValues,
boolean includeFileImplicitColumns)
Creates map with implicit columns where key is column name, value is columns actual value.
|
public ColumnExplorer(OptionManager optionManager, List<SchemaPath> columns)
public ColumnExplorer(OptionManager optionManager)
ProjectRecordBatch
.public static Map<String,ColumnExplorer.ImplicitFileColumns> initImplicitFileColumns(OptionManager optionManager)
public static Map<String,ColumnExplorer.ImplicitInternalFileColumns> initImplicitInternalFileColumns(OptionManager optionManager)
public static List<String> getImplicitColumnsNames(SchemaConfig schemaConfig)
SchemaConfig
.schemaConfig
- the source of session options values.public static boolean isPartitionColumn(OptionManager optionManager, SchemaPath column)
optionManager
- optionscolumn
- columnpublic static boolean isPartitionColumn(String partitionDesignator, String path)
partitionDesignator
- partition designatorpath
- column pathpublic boolean isImplicitOrInternalFileColumn(String name)
name
- name of the column to checktrue
if given column is implicit or internal, false
otherwisepublic static List<String> getPartitionColumnNames(FileSelection selection, ColumnNamesOptions columnNamesOptions)
selection
- the source of file pathscolumnNamesOptions
- the source of session option value for partition column labelpublic static int getPartitionDepth(FileSelection selection)
public Map<String,String> populateImplicitColumns(org.apache.hadoop.fs.Path filePath, List<String> partitionValues, boolean includeFileImplicitColumns)
filePath
- file path, used to populate file implicit columnspartitionValues
- list of partition valuesincludeFileImplicitColumns
- if file implicit columns should be included into the resultpublic Map<String,String> populateColumns(org.apache.hadoop.fs.Path filePath, List<String> partitionValues, boolean includeFileImplicitColumns, org.apache.hadoop.fs.FileSystem fs)
filePath
- file path, used to populate file implicit columnspartitionValues
- list of partition valuesincludeFileImplicitColumns
- if file implicit columns should be included into the resultfs
- file systempublic Map<String,String> populateColumns(org.apache.hadoop.fs.Path filePath, List<String> partitionValues, boolean includeFileImplicitColumns, org.apache.hadoop.fs.FileSystem fs, int index, long start, long length)
filePath
- file path, used to populate file implicit columnspartitionValues
- list of partition valuesincludeFileImplicitColumns
- if file implicit columns should be included into the resultfs
- file systemindex
- index of row group to populatestart
- start of row group to populatelength
- length of row group to populatepublic static String getImplicitColumnValue(ColumnExplorer.ImplicitFileColumn column, org.apache.hadoop.fs.Path filePath, org.apache.hadoop.fs.FileSystem fs, Integer index, Long start, Long length)
column
- implicit file columnfilePath
- file path, used to populate file implicit columnsfs
- file systemindex
- row group indexstart
- row group startlength
- row group lengthpublic static String getImplicitColumnValue(ColumnExplorer.ImplicitFileColumn column, org.apache.hadoop.fs.Path filePath, org.apache.hadoop.fs.FileSystem fs)
column
- implicit file columnfilePath
- file pathfs
- file systempublic static List<ColumnExplorer.ImplicitFileColumn> getImplicitFileColumns()
ColumnExplorer.ImplicitFileColumns
,
ColumnExplorer.ImplicitInternalFileColumns.LAST_MODIFIED_TIME
and
ColumnExplorer.ImplicitInternalFileColumns.USE_METADATA
columns.public static List<String> listPartitionValues(org.apache.hadoop.fs.Path file, org.apache.hadoop.fs.Path root, boolean hasDirsOnly)
file
- file pathroot
- root directoryhasDirsOnly
- whether it is file or directorypublic static String[] parsePartitions(org.apache.hadoop.fs.Path file, org.apache.hadoop.fs.Path root, boolean hasDirsOnly)
file
- file pathroot
- root directoryhasDirsOnly
- whether it is file or directorypublic boolean isStarQuery()
public List<SchemaPath> getTableColumns()
public boolean containsPartitionColumns()
public boolean containsImplicitColumns()
Copyright © 1970 The Apache Software Foundation. All rights reserved.