source: trunk/wikifiles/Tutorials/HadoopInASlice/hadoop_ansible.yml @ 3392

Last change on this file since 3392 was 3392, checked in by nriga@bbn.com, 6 years ago

Files for automated tutorial testing

File size: 5.6 KB
Line 
1#Copyright (c) 2015 Raytheon BBN Technologies
2#
3#Permission is hereby granted, free of charge, to any person obtaining
4#a copy of this software and/or hardware specification (the "Work") to
5#deal in the Work without restriction, including without limitation the
6#rights to use, copy, modify, merge, publish, distribute, sublicense,
7#and/or sell copies of the Work, and to permit persons to whom the Work
8#is furnished to do so, subject to the following conditions:
9#
10#The above copyright notice and this permission notice shall be
11#included in all copies or substantial portions of the Work.
12#
13#THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
14#OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
15#MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
16#NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
17#HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
18#WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19#OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
20#IN THE WORK.
21
22---
23- name: test all hosts
24  hosts: all
25  sudo: True
26  tasks:
27   - name: neuca test
28     shell: neuca-user-data
29     register: neuca_test
30   - debug: var=neuca_test.stdout_lines
31
32   - name: test if the postboot script is there
33     shell:  if [ -f /home/hadoop/hadoop_config_dynamic.sh ] ; then echo "yes" ; else echo "no" ; fi
34     register: file_test
35     failed_when: "'no' in file_test.stdout_lines"
36   - debug: var=file_test.stdout_lines
37
38   - name: install script has run
39     shell:  if  grep -q "Hello from neuca script" "/home/hadoop/hadoop_boot.log" ; then echo "yes" ; else echo "no" ; fi
40     register: install_test
41     failed_when: "'no' in install_test.stdout_lines"
42   - debug: var=install_test.stdout_lines
43     
44- name: test workers
45  hosts: worker-*
46  sudo: True
47  tasks:
48   - name: ping test
49     command: ping -c 5 master
50     register: ping_test
51   - debug: var=ping_test.stdout_lines
52
53- name: test master
54  hosts: master
55  sudo: True
56  tasks:
57
58   - name: check hadoop user is present
59     shell: "cut -d: -f1 /etc/passwd"
60     register: user_test
61     failed_when: "'hadoop' not in user_test.stdout"
62   - debug: var=user_test.stdout
63
64   - name: check formating the hadoop filesystem
65     become: True
66     become_user: hadoop
67     shell: cd ~; source ~/.bashrc; hdfs namenode -format -force
68     register: hdfs_format
69     failed_when: "'SHUTDOWN_MSG: Shutting down NameNode at master/172.16.1.1' not in hdfs_format.stderr"
70     async: 600
71     poll: 30
72   - debug: var=hdfs_format.stderr
73
74   - name: start dfs
75     become: True
76     become_user: hadoop
77     shell: cd ~; source ~/.bashrc; start-dfs.sh
78     register: dfs_start
79     async: 600
80     poll: 10
81   - debug: var=dfs_start.stderr
82
83   - name: start yarn
84     become: True
85     become_user: hadoop
86     shell: cd ~; source ~/.bashrc; start-yarn.sh
87     register: yarn_start
88     async: 600
89     poll: 10
90   - debug: var=yarn_start.stdout
91   - debug: var=yarn_start.stderr
92
93   - name: hdfs report
94     become: True
95     become_user: hadoop
96     shell: cd ~; source ~/.bashrc; hdfs dfsadmin -report
97     register: hdfs_report
98     async: 600
99     poll: 5
100   - debug: var=hdfs_report.stdout
101   - debug: var=hdfs_report.stderr
102   
103   - name: small test
104     become: True
105     become_user: hadoop
106     shell: "cd ~; source ~/.bashrc; echo Hello GENI World > /tmp/hello.txt; hdfs
107     dfs -put  /tmp/hello.txt /hello.txt; hdfs dfs -ls /; hdfs dfs -cat /hello.txt"
108     register: small_test
109     async: 600
110     poll: 5
111     failed_when: "'-rw-r--r--   2 hadoop supergroup' not in small_test.stdout"
112   - debug: var=small_test.stdout
113   - debug: var=small_test.stderr
114
115   # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798
116   # XXX When this is does remove timeout from shell hadoop command
117   - name: generate dataset
118     become: True
119     become_user: hadoop
120     shell: "cd ~; source ~/.bashrc; timeout 600 hadoop jar /home/hadoop/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar teragen 10000000 /input"
121#     async: 600
122#     poll: 20
123     register: gen_dataset
124     failed_when: "'Bytes Written=1000000000' not in gen_dataset.stderr"
125   - debug: var=gen_dataset.stderr
126   - debug: var=gen_dataset.stdout
127
128   # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798
129   # XXX When this is does remove timeout from shell hadoop command
130   - name: sort dataset
131     become: True
132     become_user: hadoop
133     shell: "cd ~; source ~/.bashrc; timeout 600 hadoop jar /home/hadoop/hadoop-2.7.1/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar terasort  /input /output"
134     register: sort_dataset
135#     async: 600
136     failed_when: "'INFO terasort.TeraSort: done' not in sort_dataset.stderr"
137   - debug: var=sort_dataset.stderr
138
139   - name: verify files
140     become: True
141     become_user: hadoop
142     shell: "cd ~; source ~/.bashrc; hdfs dfs -ls /input/; hdfs dfs -ls
143     /output/; hdfs dfs -cat /input/part-m-00000 | head; hdfs dfs -cat /output/part-r-00000 | head"
144     register: verify_files
145     async: 600
146     poll: 10
147   - debug: var=verify_files.stdout
148   - debug: var=verify_files.stderr
149
150   # Need to activate async when ansible 2.0 is out and fixes : https://github.com/ansible/ansible/issues/9798
151   # XXX When this is does remove timeout from shell execution
152   - name: hexdump
153     become: True
154     become_user: hadoop
155     shell: "cd ~; source ~/.bashrc; timeout 600 hdfs dfs -get /output/part-r-00000 /tmp/part-r-00000; hexdump /tmp/part-r-00000 | head"
156     register: hexdump
157#     async: 600
158#     poll: 30
159   - debug: var=hexdump.stdout
160   - debug: var=hexdump.stderr
161
Note: See TracBrowser for help on using the repository browser.