Skip to content

Commit 5e50873

Browse files
author
Michael R. Hines
committed
While this doesn't entirely solve our port binding issues that we're struggling to solve,
at the very least this cleans up the way we bind to ports. This commit makes all the hadoop explicitly bound to the interfaces that matters instead of the default 0.0.0.0. I've combed through all the known-used ports and tried to include all the relevant ones that are are required in CloudBench.
1 parent 75cceba commit 5e50873

File tree

1 file changed

+158
-0
lines changed

1 file changed

+158
-0
lines changed

‎scripts/hadoop/cb_config_hadoop_cluster.sh‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@ do
9696
echo "</property>" >> $output_file
9797

9898
done
99+
cat << EOF >> $output_file
100+
<property>
101+
<name>fs.ftp.host</name>
102+
<value>${my_ip_addr}</value>
103+
<description>FTP filesystem connects to this server</description>
104+
</property>
105+
EOF
99106
echo "</configuration>" >> $output_file
100107

101108
#hdfs-site.xml
@@ -113,8 +120,147 @@ do
113120
echo "</property>" >> $output_file
114121

115122
done
123+
cat << EOF >> $output_file
124+
<property>
125+
<name>dfs.namenode.secondary.http-address</name>
126+
<value>${my_ip_addr}:50090</value>
127+
<description>
128+
The secondary namenode http server address and port.
129+
</description>
130+
</property>
131+
<property>
132+
<name>dfs.namenode.secondary.https-address</name>
133+
<value>${my_ip_addr}:50091</value>
134+
<description>
135+
The secondary namenode HTTPS server address and port.
136+
</description>
137+
</property>
138+
<property>
139+
<name>dfs.datanode.address</name>
140+
<value>${my_ip_addr}:50010</value>
141+
<description>
142+
The datanode server address and port for data transfer.
143+
</description>
144+
</property>
145+
<property>
146+
<name>dfs.datanode.http.address</name>
147+
<value>${my_ip_addr}:50075</value>
148+
<description>
149+
The datanode http server address and port.
150+
</description>
151+
</property>
152+
<property>
153+
<name>dfs.datanode.ipc.address</name>
154+
<value>${my_ip_addr}:50020</value>
155+
<description>
156+
The datanode ipc server address and port.
157+
</description>
158+
</property>
159+
<property>
160+
<name>dfs.namenode.http-address</name>
161+
<value>${my_ip_addr}:50070</value>
162+
<description>
163+
The address and the base port where the dfs namenode web ui will listen on.
164+
</description>
165+
</property>
166+
<property>
167+
<name>dfs.datanode.https.address</name>
168+
<value>${my_ip_addr}:50475</value>
169+
<description>The datanode secure http server address and port.</description>
170+
</property>
171+
172+
<property>
173+
<name>dfs.namenode.https-address</name>
174+
<value>${my_ip_addr}:50470</value>
175+
<description>The namenode secure http server address and port.</description>
176+
</property>
177+
<property>
178+
<name>dfs.namenode.backup.address</name>
179+
<value>${my_ip_addr}:50100</value>
180+
<description>
181+
The backup node server address and port.
182+
If the port is 0 then the server will start on a free port.
183+
</description>
184+
</property>
185+
<property>
186+
<name>dfs.namenode.backup.http-address</name>
187+
<value>${my_ip_addr}:50105</value>
188+
<description>
189+
The backup node http server address and port.
190+
If the port is 0 then the server will start on a free port.
191+
</description>
192+
</property>
193+
<property>
194+
<name>dfs.journalnode.rpc-address</name>
195+
<value>${my_ip_addr}:8485</value>
196+
<description>
197+
The JournalNode RPC server address and port.
198+
</description>
199+
</property>
200+
201+
<property>
202+
<name>dfs.journalnode.http-address</name>
203+
<value>${my_ip_addr}:8480</value>
204+
<description>
205+
The address and port the JournalNode HTTP server listens on.
206+
If the port is 0 then the server will start on a free port.
207+
</description>
208+
</property>
209+
210+
<property>
211+
<name>dfs.journalnode.https-address</name>
212+
<value>${my_ip_addr}:8481</value>
213+
<description>
214+
The address and port the JournalNode HTTPS server listens on.
215+
If the port is 0 then the server will start on a free port.
216+
</description>
217+
</property>
218+
EOF
116219
echo "</configuration>" >> $output_file
117220

221+
#yarn-site.xml
222+
output_file=$HADOOP_CONF_DIR/yarn-site.xml
223+
sudo sed -i -e "s/<\/configuration>//" $output_file
224+
225+
cat << EOF >> $output_file
226+
<property>
227+
<description>Address where the localizer IPC is.</description>
228+
<name>yarn.nodemanager.localizer.address</name>
229+
<value>${my_ip_addr}:8040</value>
230+
</property>
231+
<property>
232+
<description>NM Webapp address.</description>
233+
<name>yarn.nodemanager.webapp.address</name>
234+
<value>${my_ip_addr}:8042</value>
235+
</property>
236+
<property>
237+
<description>The hostname of the RM.</description>
238+
<name>yarn.resourcemanager.hostname</name>
239+
<value>${my_ip_addr}</value>
240+
</property>
241+
<property>
242+
<description>The hostname of the NM.</description>
243+
<name>yarn.nodemanager.hostname</name>
244+
<value>${my_ip_addr}</value>
245+
</property>
246+
<property>
247+
<description>The hostname of the timeline service web application.</description>
248+
<name>yarn.timeline-service.hostname</name>
249+
<value>${my_ip_addr}</value>
250+
</property>
251+
<property>
252+
<description>The address of the RM admin interface.</description>
253+
<name>yarn.resourcemanager.admin.address</name>
254+
<value>${my_ip_addr}:8033</value>
255+
</property>
256+
<property>
257+
<description>The http address of the RM web application.</description>
258+
<name>yarn.resourcemanager.webapp.address</name>
259+
<value>${my_ip_addr}:8088</value>
260+
</property>
261+
</configuration>
262+
EOF
263+
118264
#mapred-site.xml
119265
output_file=$HADOOP_CONF_DIR/mapred-site.xml
120266
sudo sed -i -e "s/<\/configuration>//" $output_file
@@ -131,6 +277,18 @@ do
131277
echo "</property>" >> $output_file
132278

133279
done
280+
cat << EOF >> $output_file
281+
<property>
282+
<name>mapreduce.jobhistory.admin.address</name>
283+
<value>${my_ip_addr}:10033</value>
284+
<description>The address of the History server admin interface.</description>
285+
</property>
286+
<property>
287+
<name>mapreduce.jobhistory.webapp.address</name>
288+
<value>${my_ip_addr}:19888</value>
289+
<description>MapReduce JobHistory Server Web UI host:port</description>
290+
</property>
291+
EOF
134292
echo "</configuration>" >> $output_file
135293
syslog_netcat "...Done applying any additional Hadoop parameters."
136294

0 commit comments

Comments
 (0)