{"id":2337,"date":"2017-04-28T11:05:03","date_gmt":"2017-04-28T11:05:03","guid":{"rendered":"http:\/\/data-flair.training\/blogs\/?p=2337"},"modified":"2018-11-20T14:58:15","modified_gmt":"2018-11-20T09:28:15","slug":"apache-spark-cluster-managers-tutorial","status":"publish","type":"post","link":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/","title":{"rendered":"Apache Spark Cluster Managers &#8211; YARN, Mesos &amp; Standalone"},"content":{"rendered":"<div class='__iawmlf-post-loop-links' style='display:none;' data-iawmlf-post-links='[{&quot;id&quot;:1884,&quot;href&quot;:&quot;https:\\\/\\\/www.youtube.com\\\/watch?v=u5jA3GzZT9c&quot;,&quot;archived_href&quot;:&quot;http:\\\/\\\/web-wp.archive.org\\\/web\\\/20230418160114\\\/https:\\\/\\\/www.youtube.com\\\/watch?v=u5jA3GzZT9c&quot;,&quot;redirect_href&quot;:&quot;&quot;,&quot;checks&quot;:[{&quot;date&quot;:&quot;2025-12-10 06:37:16&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-13 18:32:24&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-20 15:40:23&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-24 02:11:15&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-27 16:42:53&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-31 16:25:19&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-04 19:36:41&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-08 21:00:46&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-13 17:00:01&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-17 02:49:02&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-20 13:23:28&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-24 04:18:09&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-27 10:25:58&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-31 20:28:55&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-05 08:43:35&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-09 14:58:01&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-12 15:52:10&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-16 21:33:51&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-23 22:27:37&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-27 17:59:40&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-03 09:57:38&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-07 00:49:48&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-11 07:25:31&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-16 03:29:19&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-22 01:05:26&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-30 17:41:12&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-04 12:43:52&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-07 16:24:17&quot;,&quot;http_code&quot;:404},{&quot;date&quot;:&quot;2026-04-10 21:58:52&quot;,&quot;http_code&quot;:404},{&quot;date&quot;:&quot;2026-04-15 13:49:50&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-19 18:38:59&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-24 06:16:55&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-27 18:34:47&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-02 05:11:21&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-07 12:47:12&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-11 17:49:32&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-17 09:22:03&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-24 07:45:15&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-28 12:28:13&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-06-02 02:10:20&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-06-05 10:03:42&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-06-09 10:47:41&quot;,&quot;http_code&quot;:206}],&quot;broken&quot;:false,&quot;last_checked&quot;:{&quot;date&quot;:&quot;2026-06-09 10:47:41&quot;,&quot;http_code&quot;:206},&quot;process&quot;:&quot;done&quot;},{&quot;id&quot;:2359,&quot;href&quot;:&quot;https:\\\/\\\/spark.apache.org\\\/docs\\\/2.3.0&quot;,&quot;archived_href&quot;:&quot;http:\\\/\\\/web-wp.archive.org\\\/web\\\/20250905123917\\\/https:\\\/\\\/spark.apache.org\\\/docs\\\/2.3.0\\\/&quot;,&quot;redirect_href&quot;:&quot;&quot;,&quot;checks&quot;:[{&quot;date&quot;:&quot;2025-12-11 04:18:01&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-14 14:28:21&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-17 19:55:17&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-21 05:47:18&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-24 05:53:07&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-27 11:04:43&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2025-12-30 17:18:39&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-02 18:43:42&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-06 02:01:41&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-09 07:10:37&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-12 20:30:05&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-16 05:06:04&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-19 07:59:15&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-22 09:39:59&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-25 15:25:25&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-01-29 05:12:15&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-02 05:12:24&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-05 08:43:37&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-08 19:58:36&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-11 21:35:21&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-15 18:46:08&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-19 11:58:38&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-23 01:53:59&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-02-26 08:15:15&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-02 11:04:18&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-05 19:03:56&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-09 01:21:19&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-12 04:00:55&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-16 06:21:54&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-19 16:11:32&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-23 13:45:15&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-28 13:10:35&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-03-31 19:02:44&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-04 09:30:06&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-07 13:45:17&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-10 15:34:26&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-14 07:09:45&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-17 07:20:17&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-20 19:08:35&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-24 18:20:36&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-04-28 10:19:27&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-02 09:09:29&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-05 21:34:04&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-09 08:44:57&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-12 17:54:45&quot;,&quot;http_code&quot;:503},{&quot;date&quot;:&quot;2026-05-17 09:22:05&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-21 21:30:08&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-25 02:23:07&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-05-30 05:07:01&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-06-02 11:39:04&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-06-05 22:12:57&quot;,&quot;http_code&quot;:206},{&quot;date&quot;:&quot;2026-06-11 09:37:32&quot;,&quot;http_code&quot;:206}],&quot;broken&quot;:false,&quot;last_checked&quot;:{&quot;date&quot;:&quot;2026-06-11 09:37:32&quot;,&quot;http_code&quot;:206},&quot;process&quot;:&quot;done&quot;}]'><\/div>\n<h2>1. Objective<\/h2>\n<p>Today, in this tutorial on <strong><a href=\"http:\/\/data-flair.training\/blogs\/apache-spark-tutorial-quickstart-introduction\/\">Apache Spark<\/a><\/strong> cluster managers, we are going to learn what Cluster Manager in Spark is. Moreover, we will discuss various types of cluster managers-<strong>Spark Standalone cluster, YARN mode, and Spark Mesos<\/strong>. Also, we will learn how Apache Spark cluster managers work. The difference between Spark Standalone vs YARN vs Mesos is also covered in this blog.<\/p>\n<p>So, let&#8217;s start Spark ClustersManagerss tutorial.<\/p>\n<div id=\"attachment_42937\" style=\"width: 1210px\" class=\"wp-caption aligncenter\"><a href=\"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg\"><img loading=\"lazy\" decoding=\"async\" aria-describedby=\"caption-attachment-42937\" class=\"size-full wp-image-42937\" src=\"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg\" alt=\"Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone\" width=\"1200\" height=\"628\" srcset=\"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg 1200w, https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1-150x79.jpg 150w, https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1-300x157.jpg 300w, https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1-768x402.jpg 768w, https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1-1024x536.jpg 1024w, https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1-520x272.jpg 520w\" sizes=\"auto, (max-width: 1200px) 100vw, 1200px\" \/><\/a><p id=\"caption-attachment-42937\" class=\"wp-caption-text\">Apache Spark Cluster Managers &#8211; YARN, Mesos &amp; Standalone<\/p><\/div>\n<h2>2. Introduction to Apache Spark Cluster Managers<\/h2>\n<p><strong>Apache Spark<\/strong> is an engine for <a href=\"http:\/\/data-flair.training\/blogs\/why-learn-big-data-use-cases\/\"><strong>Big Data<\/strong><\/a> processing. One can run Spark on distributed mode on the cluster. In the cluster, there is a master and n number of workers. It schedules and divides resource in the host machine which forms the cluster. The prime work of the cluster manager is to divide resources across applications. It works as an external service for acquiring resources on the cluster.<\/p>\n<p>The cluster manager dispatches work for the cluster. Spark supports pluggable cluster management. The cluster manager in Spark handles starting executor processes. Refer this link to <a href=\"http:\/\/data-flair.training\/blogs\/important-apache-spark-terminologies-and-concepts-you-must-know\/\">learn Apache Spark terminologies and concepts<\/a>.<\/p>\n<p>Apache Spark system supports three types of cluster managers namely-<\/p>\n<p>a) Standalone Cluster Manager<\/p>\n<p>b) Hadoop YARN<\/p>\n<p>c) Apache Mesos<\/p>\n<p>So, let&#8217;s discuss these Apache Spark Cluster Managers in detail.<\/p>\n<h3>i. Apache Spark Standalone Cluster Manager<\/h3>\n<p><strong>Standalone mode<\/strong> is a simple cluster manager incorporated with Spark. It makes it easy to setup a cluster that Spark itself manages and can run on<strong> <a href=\"http:\/\/data-flair.training\/blogs\/frequent-linux-commands-beginners-tutorial\/\">Linux<\/a><\/strong>, Windows, or Mac OSX. Often it is the simplest way to run Spark application in a clustered environment. Learn,\u00a0<a href=\"http:\/\/data-flair.training\/blogs\/apache-spark-installation-in-standalone-mode\/\">how to install Apache Spark On Standalone Mode<\/a>.<\/p>\n<h4>a. How does Spark Standalone Cluster Works?<\/h4>\n<p>It has <strong><em>masters<\/em><\/strong> and number of<strong> <em>workers<\/em> <\/strong>with configured amount of memory and CPU cores. In Spark standalone cluster mode, Spark allocates resources based on the core. By default, an application will grab all the cores in the cluster.<\/p>\n<p>In standalone cluster manager, <strong>Zookeeper quorum<\/strong>\u00a0recovers the master using <em>standby master.<\/em> Using the file system, we can achieve the manual recovery of the master. Spark supports authentication with the help of shared secret with entire cluster manager. The user configures each node with a shared secret. For communication protocols, Data encrypts using SSL. But for block transfer, it makes use of data SASL encryption.<\/p>\n<p>To check the application, each Apache Spark application has a Web User Interface. The Web UI provides information of executors, storage usage, running task in the application. In this cluster manager, we have Web UI to view cluster and job statistics. It also has detailed log output for each job. If an application has logged event for its lifetime, Spark Web UI will reconstruct the application\u2019s UI after the application exits.<\/p>\n<h3>ii. Apache Mesos<\/h3>\n<p><strong>Mesos<\/strong> handles the workload in distributed environment by dynamic resource sharing and isolation. It is healthful for deployment and management of applications in large-scale cluster environments. Apache Mesos clubs together the existing resource of the machines\/nodes in a cluster. From this, a variety of workloads may use. This is node abstraction, thus it decreases an overhead of allocating a specific machine for different workloads. It is resource management platform for <strong><a href=\"http:\/\/data-flair.training\/blogs\/hadoop-introduction-tutorial-quick-guide\/\">Hadoop<\/a> <\/strong>and<strong> <a href=\"http:\/\/data-flair.training\/blogs\/big-data-history-use-cases\/\">Big Data<\/a><\/strong> cluster.<\/p>\n<p>Companies such as <em>Twitter<\/em>, <em>Xogito<\/em>, and <em>Airbnb<\/em> use Apache Mesos as it can run on Linux or Mac OSX.<\/p>\n<p>In some way, Apache Mesos is the reverse of <em>virtualization<\/em>. This is because in virtualization one physical resource divides into many virtual resources. While in Mesos many physical resources are club into a single virtual resource. Refer this link to <a href=\"http:\/\/data-flair.training\/blogs\/apache-mesos-tutorial-learn-mesos\/\">learn Apache Mesos in detail<\/a>.<\/p>\n<h4>a. How do Apache Mesos works?<\/h4>\n<p>The three components of Apache Mesos are <em>Mesos masters<\/em>, <em>Mesos slave<\/em>, <em>Frameworks<\/em>.<\/p>\n<p><strong>Mesos Master<\/strong> is an instance of the cluster. A cluster has many Mesos masters that provide <a href=\"http:\/\/data-flair.training\/blogs\/apache-spark-streaming-fault-tolerance\/\">fault tolerance<\/a>. Here one instance is the leading master. <strong>Mesos Slave<\/strong> is Mesos instance that offers resources to the cluster. Mesos Master assigns the task to the slave. <strong>Mesos Framework<\/strong> allows applications to request the resources from the cluster. Thus, the application can perform the task.<\/p>\n<p>Some other Frameworks by Mesos are <em>Chronos<\/em>, <em>Marathon<\/em>, <em>Aurora<\/em>, <em><a href=\"https:\/\/www.youtube.com\/watch?v=u5jA3GzZT9c\">Hadoop<\/a><\/em>, <em><a href=\"http:\/\/data-flair.training\/blogs\/apache-spark-introduction-spark-comprehensive-tutorial\/\">Spark<\/a><\/em>, <em>Jenkins<\/em> etc.<\/p>\n<p><strong>Mesos authentication includes:<\/strong><\/p>\n<ul>\n<li>The slave\u2019s registration with the master.<\/li>\n<li>Frameworks (that is, applications) submission to the cluster.<\/li>\n<li>Operators using endpoints such as HTTP endpoints.<\/li>\n<\/ul>\n<p><a href=\"http:\/\/data-flair.training\/blogs\/apache-mesos-books-mastering-mesos\/\">Get the best Apache Mesos books to master Mesos<\/a>.<\/p>\n<h3>iii. Hadoop YARN<\/h3>\n<p><strong>YARN<\/strong> became the sub-project of <a href=\"http:\/\/data-flair.training\/blogs\/important-big-data-terminologies-and-hadoop-concepts-you-must-know\/\">Hadoop<\/a> in the year 2012. It is also known as <strong><a href=\"http:\/\/data-flair.training\/blogs\/hadoop-mapreduce-introduction-tutorial-comprehensive-guide\/\">MapReduce<\/a><\/strong> <strong>2.0. <\/strong>YARN bifurcate the functionality of <em>resource manager<\/em> and <em>job scheduling<\/em> into different daemons. The plan is to get a Global Resource Manager (RM) and per-application Application Master (AM). An application is either a<strong><a href=\"http:\/\/data-flair.training\/blogs\/directed-acyclic-graph-dag-in-apache-spark\/\"> DAG<\/a><\/strong> of graph or an individual job.<\/p>\n<p>To learn YARN is great detail <a href=\"http:\/\/data-flair.training\/blogs\/hadoop-yarn-tutorial\/\">follow this Yarn tutorial<\/a>.<\/p>\n<h4>a. How do Hadoop YARN works?<\/h4>\n<p><strong>YARN<\/strong> data computation framework is a combination of the <em>ResourceManager<\/em>, the <em>NodeManager<\/em>. It can run on <a href=\"http:\/\/data-flair.training\/blogs\/frequently-used-important-linux-commands-tutorial\/\">Linux<\/a> and Windows.<\/p>\n<p><a href=\"http:\/\/data-flair.training\/blogs\/hadoop-yarn-resource-manager-guide-tutorial\/\"><strong>The Yarn Resource Manager<\/strong> <\/a>manages resources among all the applications in the system. The Resource Manager has scheduler and Application Manager. <strong>The Scheduler<\/strong> allocates resource to the various running application. It is pure Scheduler, performs monitoring or tracking of status for the application. <strong>The Application Manager<\/strong> manages applications across all the nodes.<\/p>\n<p><a href=\"http:\/\/data-flair.training\/blogs\/hadoop-yarn-node-manager-tutorial-guide\/\"><strong>Yarn Node Manager<\/strong><\/a> contains Application Master and container. <strong>A container<\/strong> is a place where a unit of work happens. Each task of MapReduce runs in one container. The per-application <strong>Application Master<\/strong> is a framework specific library. It aims to negotiate resources from the Resource Manager. It continues with Node Manager(s) to execute and watch the tasks.<\/p>\n<p>The application or job requires one or more containers. Node Manager handles monitoring containers, resource usage (CPU, memory, disk, and network). It reports this to the Resource Manager. YARN provides security for authentication, service level authorization. It also provides authentication for Web consoles and data confidentiality. Hadoop authentication uses <em>Kerberos<\/em> to verify that each user and service has authentication.<\/p>\n<h2>3. Spark Standalone mode vs YARN vs Mesos<\/h2>\n<p>In this tutorial of Apache Spark Cluster Managers, features of 3 modes of Spark cluster have already present. Let us now see the comparison between Standalone mode vs YARN cluster vs Mesos Cluster in Apache Spark in details. It will help you to understand which Apache Spark Cluster Managers\u00a0type one should choose for Spark.<\/p>\n<h3>i. High Availability<\/h3>\n<p>The <strong>standalone cluster<\/strong>: with <em>ZooKeeper Quorum<\/em> it supports an automatic recovery of the master. One can achieve manual recovery using the file system. The cluster tolerates the worker failure despite the recovery of the Master is enabling or not.<\/p>\n<p>The <strong>Apache Mesos<\/strong>: using Apache ZooKeeper it supports an automatic recovery of the master. In the case of failover, tasks which are currently executing, do not stop their execution.<\/p>\n<p><strong>Apache Hadoop YARN<\/strong>: using a command line utility it supports manual recovery. And use Zookeeper-based <em>ActiveStandbyElector<\/em> embedded in the ResourceManager for automatic recovery. Thus, there is no need to run a separate ZooKeeper Failover Controller.<\/p>\n<h3>ii. Security<\/h3>\n<p>Spark supports authentication via a shared secret with all the cluster managers. The <strong>standalone manager<\/strong> requires the user to configure each of the nodes with the shared secret. Data can be encrypted using SSL for the communication protocols. SASL encryption is supported for block transfers of data. Other options are also available for encrypting data. Access to Spark applications in the Web UI can be controlled via access control lists.<\/p>\n<p><strong>Mesos:<\/strong> For any entity interacting with the cluster Mesos provides authentication. This includes the slaves registering with the master, frameworks submitted to the cluster, and operators using endpoints such as HTTP endpoints. These entities can be enabling to use authentication or not. Custom module can replace Mesos\u2019 default authentication module, Cyrus SASL.<\/p>\n<p>To allow access to services in Mesos Access, it makes use of control lists. By default, communication between the modules in Mesos is unencrypted. SSL\/TLS can be enabled to encrypt communication. Mesos WebUI supports HTTPS.<\/p>\n<p><strong>Hadoop YARN:<\/strong> It contains security for authentication, service level authorization, authentication for Web consoles and data confidentiality. Using Service level authorization it ensures that client using Hadoop services has authority. Using access control lists Hadoop services can be controlled. Additionally, using SSL data and communication between clients and services is encrypted. The data transferred between the Web console and clients with HTTPS.<\/p>\n<h3>iii. Monitoring<\/h3>\n<p><strong>Spark\u2019s standalone cluster manager<\/strong>: To view cluster and job statistics it has a Web UI. It also has detailed log output for each job. the Spark Web UI will reconstruct the application\u2019s UI after the application exists if an application has logged events for its lifetime.<\/p>\n<p><strong>Apache Mesos: <\/strong>It supports per container network monitoring and isolation. It provides many metrics for <em>master<\/em> and<em> slave<\/em> nodes accessible with URL. These metrics include percentage and number of allocated CPU&#8217;s, memory usage etc.<\/p>\n<p><strong>Hadoop YARN<\/strong> has a Web UI for the <em>ResourceManager<\/em> and the <em>NodeManager<\/em>. The ResourceManager UI provides metrics for the cluster. And the NodeManager provides information for each node, the applications and containers running on the node.<\/p>\n<h2>4. Conclusion<\/h2>\n<p>Hence, in this Apache Spark Cluster Managers tutorial, we can say Standalone mode is easy to set up among all. It will provide almost all the same features as the other cluster managers.<\/p>\n<p>Moreover, to use richer resource scheduling capabilities (e.g. queues), both YARN and Mesos provide these features. Of these, YARN allows you to share and configure the same pool of cluster resources between all frameworks that run on YARN. It is likely to be pre-installed on Hadoop systems.<\/p>\n<p>Also, one advantage of Mesos over both YARN and the standalone mode is its fine-grained sharing option. Now, this lets interactive applications (Spark shell) scale down their CPU allocation between commands. This makes it attractive in environments where many users are running interactive shells.<\/p>\n<p>If you like this blog or have any query about Apache Spark Cluster Managers, so, do let us know by leaving a comment.<\/p>\n<p><strong>See Also-<\/strong><\/p>\n<ul>\n<li><a href=\"http:\/\/data-flair.training\/blogs\/comparison-between-apache-mesos-vs-hadoop-yarn\/\">Hadoop Yarn vs Apache Mesos<\/a><\/li>\n<li><a href=\"http:\/\/data-flair.training\/blogs\/install-apache-spark-multi-node-cluster\/\">Apache Spark multi-node cluster setup.<\/a><\/li>\n<\/ul>\n<p><strong><a href=\"https:\/\/spark.apache.org\/docs\/2.3.0\/\">Reference for Spark<\/a><\/strong><\/p>\n","protected":false},"excerpt":{"rendered":"<p>1. Objective Today, in this tutorial on Apache Spark cluster managers, we are going to learn what Cluster Manager in Spark is. Moreover, we will discuss various types of cluster managers-Spark Standalone cluster, YARN&#46;&#46;&#46;<\/p>\n","protected":false},"author":6,"featured_media":42937,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[10],"tags":[904,911,2601,2602,13040,13128],"class_list":["post-2337","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-spark","tag-apache-spark-cluster-manager","tag-apache-spark-deployment-mode","tag-cluster-manager-in-apache-spark","tag-cluster-manager-in-spark","tag-spark-cluster-manager","tag-spark-standalone-mode"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v27.4 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone - DataFlair<\/title>\n<meta name=\"description\" content=\"Cluster manager in Spark,types of Spark cluster manager,Spark Standalone cluster, YARN mode,Spark Mesos,difference between Spark standalone vs YARN vs Mesos\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone - DataFlair\" \/>\n<meta property=\"og:description\" content=\"Cluster manager in Spark,types of Spark cluster manager,Spark Standalone cluster, YARN mode,Spark Mesos,difference between Spark standalone vs YARN vs Mesos\" \/>\n<meta property=\"og:url\" content=\"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/\" \/>\n<meta property=\"og:site_name\" content=\"DataFlair\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/DataFlairWS\/\" \/>\n<meta property=\"article:published_time\" content=\"2017-04-28T11:05:03+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2018-11-20T09:28:15+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg\" \/>\n\t<meta property=\"og:image:width\" content=\"1200\" \/>\n\t<meta property=\"og:image:height\" content=\"628\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/jpeg\" \/>\n<meta name=\"author\" content=\"DataFlair Team\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:creator\" content=\"@DataFlairWS\" \/>\n<meta name=\"twitter:site\" content=\"@DataFlairWS\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"DataFlair Team\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"8 minutes\" \/>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone - DataFlair","description":"Cluster manager in Spark,types of Spark cluster manager,Spark Standalone cluster, YARN mode,Spark Mesos,difference between Spark standalone vs YARN vs Mesos","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/","og_locale":"en_US","og_type":"article","og_title":"Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone - DataFlair","og_description":"Cluster manager in Spark,types of Spark cluster manager,Spark Standalone cluster, YARN mode,Spark Mesos,difference between Spark standalone vs YARN vs Mesos","og_url":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/","og_site_name":"DataFlair","article_publisher":"https:\/\/www.facebook.com\/DataFlairWS\/","article_published_time":"2017-04-28T11:05:03+00:00","article_modified_time":"2018-11-20T09:28:15+00:00","og_image":[{"width":1200,"height":628,"url":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg","type":"image\/jpeg"}],"author":"DataFlair Team","twitter_card":"summary_large_image","twitter_creator":"@DataFlairWS","twitter_site":"@DataFlairWS","twitter_misc":{"Written by":"DataFlair Team","Est. reading time":"8 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#article","isPartOf":{"@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/"},"author":{"name":"DataFlair Team","@id":"https:\/\/data-flair.training\/blogs\/#\/schema\/person\/2c58ecb4f73a39f0ef993f1ddfcd7b89"},"headline":"Apache Spark Cluster Managers &#8211; YARN, Mesos &amp; Standalone","datePublished":"2017-04-28T11:05:03+00:00","dateModified":"2018-11-20T09:28:15+00:00","mainEntityOfPage":{"@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/"},"wordCount":1680,"commentCount":0,"publisher":{"@id":"https:\/\/data-flair.training\/blogs\/#organization"},"image":{"@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#primaryimage"},"thumbnailUrl":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg","keywords":["Apache Spark cluster manager","Apache Spark Deployment mode","cluster manager in Apache Spark","cluster manager in Spark","Spark cluster manager","spark standalone mode"],"articleSection":["Apache Spark Tutorials"],"inLanguage":"en-US","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#respond"]}]},{"@type":"WebPage","@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/","url":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/","name":"Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone - DataFlair","isPartOf":{"@id":"https:\/\/data-flair.training\/blogs\/#website"},"primaryImageOfPage":{"@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#primaryimage"},"image":{"@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#primaryimage"},"thumbnailUrl":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg","datePublished":"2017-04-28T11:05:03+00:00","dateModified":"2018-11-20T09:28:15+00:00","description":"Cluster manager in Spark,types of Spark cluster manager,Spark Standalone cluster, YARN mode,Spark Mesos,difference between Spark standalone vs YARN vs Mesos","breadcrumb":{"@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#primaryimage","url":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg","contentUrl":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2017\/04\/apache-spark-cluster-managers-yarn-mesos-standalone-768x402-1.jpg","width":1200,"height":628,"caption":"Apache Spark Cluster Managers - YARN, Mesos &amp; Standalone"},{"@type":"BreadcrumbList","@id":"https:\/\/data-flair.training\/blogs\/apache-spark-cluster-managers-tutorial\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Blog Home","item":"https:\/\/data-flair.training\/blogs\/"},{"@type":"ListItem","position":2,"name":"Apache Spark Tutorials","item":"https:\/\/data-flair.training\/blogs\/category\/spark\/"},{"@type":"ListItem","position":3,"name":"Apache Spark Cluster Managers &#8211; YARN, Mesos &amp; Standalone"}]},{"@type":"WebSite","@id":"https:\/\/data-flair.training\/blogs\/#website","url":"https:\/\/data-flair.training\/blogs\/","name":"DataFlair","description":"Learn Today. Lead Tomorrow.","publisher":{"@id":"https:\/\/data-flair.training\/blogs\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/data-flair.training\/blogs\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/data-flair.training\/blogs\/#organization","name":"DataFlair","url":"https:\/\/data-flair.training\/blogs\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/data-flair.training\/blogs\/#\/schema\/logo\/image\/","url":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2016\/07\/Data-Flair.png","contentUrl":"https:\/\/data-flair.training\/blogs\/wp-content\/uploads\/sites\/2\/2016\/07\/Data-Flair.png","width":106,"height":48,"caption":"DataFlair"},"image":{"@id":"https:\/\/data-flair.training\/blogs\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/DataFlairWS\/","https:\/\/x.com\/DataFlairWS","https:\/\/www.linkedin.com\/company\/dataflair-web-services-pvt-ltd\/","https:\/\/www.youtube.com\/user\/DataFlairWS"]},{"@type":"Person","@id":"https:\/\/data-flair.training\/blogs\/#\/schema\/person\/2c58ecb4f73a39f0ef993f1ddfcd7b89","name":"DataFlair Team","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/1ce4a0e3e542444fc73bbebf83e89e8b73e2d95ccb1fcee64da9945f078b97c5?s=96&d=mm&r=g","url":"https:\/\/secure.gravatar.com\/avatar\/1ce4a0e3e542444fc73bbebf83e89e8b73e2d95ccb1fcee64da9945f078b97c5?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/1ce4a0e3e542444fc73bbebf83e89e8b73e2d95ccb1fcee64da9945f078b97c5?s=96&d=mm&r=g","caption":"DataFlair Team"},"description":"The DataFlair Team provides industry-driven content on programming, Java, Python, C++, DSA, AI, ML, data Science, Android, Flutter, MERN, Web Development, and technology. Our expert educators focus on delivering value-packed, easy-to-follow resources for tech enthusiasts and professionals.","url":"https:\/\/data-flair.training\/blogs\/author\/dfteam2\/"}]}},"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/posts\/2337","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/users\/6"}],"replies":[{"embeddable":true,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/comments?post=2337"}],"version-history":[{"count":6,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/posts\/2337\/revisions"}],"predecessor-version":[{"id":42939,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/posts\/2337\/revisions\/42939"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/media\/42937"}],"wp:attachment":[{"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/media?parent=2337"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/categories?post=2337"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/data-flair.training\/blogs\/wp-json\/wp\/v2\/tags?post=2337"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}