Puppet Class: hadoop::create_dirs

Defined in:
manifests/create_dirs.pp

Overview

Class hadoop::create_dirs

Create root directory layout at Hadoop Filesystem. Take care also for Kerberos ticket inicialization and destruction, when realm is specified.

Requirements: running HDFS. Either it needs to be formatted and namenode service running, or external storage needs to be used.

This class is called from hadoop::service.



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'manifests/create_dirs.pp', line 9

class hadoop::create_dirs {
  $realm = $hadoop::realm
  # hadoop.security.auth_to_local not used by ResourceManager state-store
  # in Hadoop 2.4.1 in Fedora
  if ($realm and $realm != '' and "${::osfamily}/${::operatingsystem}" == 'RedHat/Fedora') { $rmstore_user = 'rm' }
  else { $rmstore_user = 'yarn' }

  hadoop::kinit { 'hdfs-kinit':
    touchfile => 'hdfs-root-created',
  }
  ->
  hadoop::mkdir { '/user':
    touchfile => 'hdfs-root-created',
  }
  ->
  hadoop::mkdir { '/var/log':
    touchfile => 'hdfs-root-created',
  }
  ->
  hadoop::mkdir { '/tmp':
    mode      => '1777',
    touchfile => 'hdfs-root-created',
  }
  ->
  hadoop::mkdir { ['/tmp/hadoop-yarn/staging', '/tmp/hadoop-yarn/staging/history', '/tmp/hadoop-yarn/staging/history/done', '/tmp/hadoop-yarn/staging/history/done_intermediate']:
    mode      => '1777',
    owner     => 'mapred',
    group     => 'mapred',
    touchfile => 'hdfs-root-created',
  }
  ->
  hadoop::mkdir { ['/tmp/hadoop-yarn', '/var/log/hadoop-yarn']:
    owner     => 'yarn',
    group     => 'mapred',
    touchfile => 'hdfs-root-created',
  }
  ->
  # log aggregation needs access for both: yarn and mapred (not needed for
  # CDH 5.4.2, needed for CDH 5.4.7)
  hadoop::mkdir { '/var/log/hadoop-yarn/apps':
    mode      => '1777',
    owner     => 'yarn',
    group     => 'hadoop',
    touchfile => 'hdfs-root-created',
  }
  ->
  # for resource manager state store feature
  hadoop::mkdir { '/rmstore':
    owner     => $rmstore_user,
    group     => 'hadoop',
    touchfile => 'hdfs-root-created',
  }
  ->
  hadoop::kdestroy { 'hdfs-kdestroy':
    touchfile => 'hdfs-root-created',
    touch     => true,
  }
}