Source code for luigi.contrib.hdfs.abstract_client

# -*- coding: utf-8 -*-
# Copyright 2012-2015 Spotify AB
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

Module containing abstract class about hdfs clients.

import abc
from luigi import six

[docs]@six.add_metaclass(abc.ABCMeta) class HdfsFileSystem( """ This client uses Apache 2.x syntax for file system commands, which also matched CDH4. """
[docs] def rename(self, path, dest): """ Rename or move a file. In hdfs land, "mv" is often called rename. So we add an alias for ``move()`` called ``rename()``. This is also to keep backward compatibility since ``move()`` became standardized in luigi's filesystem interface. """ return self.move(path, dest)
[docs] def rename_dont_move(self, path, dest): """ Override this method with an implementation that uses rename2, which is a rename operation that never moves. rename2 - (lines 483-523) """ # We only override this method to be able to provide a more specific # docstring. return super(HdfsFileSystem, self).rename_dont_move(path, dest)
[docs] @abc.abstractmethod def remove(self, path, recursive=True, skip_trash=False): pass
[docs] @abc.abstractmethod def chmod(self, path, permissions, recursive=False): pass
[docs] @abc.abstractmethod def chown(self, path, owner, group, recursive=False): pass
[docs] @abc.abstractmethod def count(self, path): """ Count contents in a directory """ pass
[docs] @abc.abstractmethod def copy(self, path, destination): pass
[docs] @abc.abstractmethod def put(self, local_path, destination): pass
[docs] @abc.abstractmethod def get(self, path, local_destination): pass
[docs] @abc.abstractmethod def mkdir(self, path, parents=True, raise_if_exists=False): pass
[docs] @abc.abstractmethod def listdir(self, path, ignore_directories=False, ignore_files=False, include_size=False, include_type=False, include_time=False, recursive=False): pass
[docs] @abc.abstractmethod def touchz(self, path): pass