Source code for luigi.contrib.hdfs.abstract_client
# -*- coding: utf-8 -*-
#
# Copyright 2012-2015 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Module containing abstract class about hdfs clients.
"""
import abc
import luigi.target
[docs]
class HdfsFileSystem(luigi.target.FileSystem, metaclass=abc.ABCMeta):
"""
This client uses Apache 2.x syntax for file system commands, which also matched CDH4.
"""
[docs]
def rename(self, path, dest):
"""
Rename or move a file.
In hdfs land, "mv" is often called rename. So we add an alias for
``move()`` called ``rename()``. This is also to keep backward
compatibility since ``move()`` became standardized in luigi's
filesystem interface.
"""
return self.move(path, dest)
[docs]
def rename_dont_move(self, path, dest):
"""
Override this method with an implementation that uses rename2,
which is a rename operation that never moves.
rename2 -
https://github.com/apache/hadoop/blob/ae91b13/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
(lines 483-523)
"""
# We only override this method to be able to provide a more specific
# docstring.
return super(HdfsFileSystem, self).rename_dont_move(path, dest)
[docs]
@abc.abstractmethod
def remove(self, path, recursive=True, skip_trash=False):
pass
[docs]
@abc.abstractmethod
def chmod(self, path, permissions, recursive=False):
pass
[docs]
@abc.abstractmethod
def chown(self, path, owner, group, recursive=False):
pass
[docs]
@abc.abstractmethod
def count(self, path):
"""
Count contents in a directory
"""
pass
[docs]
@abc.abstractmethod
def copy(self, path, destination):
pass
[docs]
@abc.abstractmethod
def put(self, local_path, destination):
pass
[docs]
@abc.abstractmethod
def get(self, path, local_destination):
pass
[docs]
@abc.abstractmethod
def mkdir(self, path, parents=True, raise_if_exists=False):
pass
[docs]
@abc.abstractmethod
def listdir(self, path, ignore_directories=False, ignore_files=False,
include_size=False, include_type=False, include_time=False, recursive=False):
pass
[docs]
@abc.abstractmethod
def touchz(self, path):
pass