xrootd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
XrdCmsCluster.hh
Go to the documentation of this file.
1 #ifndef __CMS_CLUSTER__H
2 #define __CMS_CLUSTER__H
3 /******************************************************************************/
4 /* */
5 /* X r d C m s C l u s t e r . h h */
6 /* */
7 /* (c) 2007 by the Board of Trustees of the Leland Stanford, Jr., University */
8 /* All Rights Reserved */
9 /* Produced by Andrew Hanushevsky for Stanford University under contract */
10 /* DE-AC02-76-SFO0515 with the Department of Energy */
11 /* */
12 /* This file is part of the XRootD software suite. */
13 /* */
14 /* XRootD is free software: you can redistribute it and/or modify it under */
15 /* the terms of the GNU Lesser General Public License as published by the */
16 /* Free Software Foundation, either version 3 of the License, or (at your */
17 /* option) any later version. */
18 /* */
19 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
20 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
21 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
22 /* License for more details. */
23 /* */
24 /* You should have received a copy of the GNU Lesser General Public License */
25 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
26 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
27 /* */
28 /* The copyright holder's institutional names and contributor's names may not */
29 /* be used to endorse or promote products derived from this software without */
30 /* specific prior written permission of the institution or contributor. */
31 /******************************************************************************/
32 
33 #include <cstdlib>
34 #include <cstring>
35 #include <strings.h>
36 #include <netinet/in.h>
37 
38 #include "XrdCms/XrdCmsTypes.hh"
39 #include "XrdOuc/XrdOucTList.hh"
40 #include "XrdOuc/XrdOucEnum.hh"
41 #include "XrdSys/XrdSysPthread.hh"
42 #include "XrdSys/XrdSysRAtomic.hh"
43 
44 class XrdLink;
45 class XrdCmsDrop;
46 class XrdCmsNode;
47 class XrdCmsSelect;
48 class XrdCmsSelector;
49 class XrdNetAddr;
50 
51 namespace XrdCms
52 {
53 struct CmsRRHdr;
54 }
55 
56 /******************************************************************************/
57 /* O p t i o n F l a g s */
58 /******************************************************************************/
59 
60 namespace XrdCms
61 {
62 
63 // Flags passed to Add()
64 //
65 static const int CMS_noStage = 1;
66 static const int CMS_Suspend = 2;
67 static const int CMS_Perm = 4;
68 static const int CMS_isMan = 8;
69 static const int CMS_Lost = 16;
70 static const int CMS_isPeer = 32;
71 static const int CMS_isProxy = 64;
72 static const int CMS_noSpace =128;
73 static const int CMS_isSuper =256;
74 
75 static const int CMS_isVers3 =0x01000000;
76 
78 static const int CMS_hasAlts =CMS_isMan|CMS_isPeer;
79 
80 // Class passed to Space()
81 //
82 class SpaceData
83 {
84 public:
85 
86 long long Total; // Total space
87 long long TotFr; // Total space free
88 int wMinF; // Free space minimum to select wFree node
89 int wFree; // Free space for nodes providing r/w access (largest one)
90 int wNum; // Number of nodes providing r/w access
91 int wUtil; // Average utilization (largest one)
92 int sFree; // Free space for nodes providing staging (largest one)
93 int sNum; // Number of nodes providing staging
94 int sUtil; // Average utilization (largest one)
95 
96  SpaceData() : Total(0), TotFr(0),wMinF(0),
97  wFree(0), wNum(0), wUtil(0),
98  sFree(0), sNum(0), sUtil(0) {}
100 };
101 }
102 
103 /******************************************************************************/
104 /* C l a s s X r d C m s C l u s t e r */
105 /******************************************************************************/
106 
107 // This a single-instance global class
108 //
109 class XrdCmsBaseFR;
110 class XrdCmsClustID;
111 class XrdCmsSelected;
112 class XrdOucTList;
113 
115 {
116 public:
117 friend class XrdCmsDrop;
118 
119 int NodeCnt; // Number of active nodes
120 
121 // Called to add a new node to the cluster. Status values are defined above.
122 //
123 XrdCmsNode *Add(XrdLink *lp, int dport, int Status,
124  int sport, const char *theNID, const char *theIF);
125 
126 // Put nodes in or remove from a blacklist
127 //
128 virtual void BlackList(XrdOucTList *blP);
129 
130 // Sends a message to all nodes matching smask (three forms for convenience)
131 //
132 SMask_t Broadcast(SMask_t, const struct iovec *, int, int tot=0);
133 
135  char *Data, int Dlen=0);
136 
138  void *Data, int Dlen);
139 
140 // Sends a message to a single node in a round-robbin fashion.
141 //
142 int Broadsend(SMask_t smask, XrdCms::CmsRRHdr &Hdr,
143  void *Data, int Dlen);
144 
145 // Returns the node mask matching the given IP address
146 //
147 SMask_t getMask(const XrdNetAddr *addr);
148 
149 // Returns the node mask matching the given cluster ID
150 //
151 SMask_t getMask(const char *Cid);
152 
153 // Extracts out node information. Opts are one or more of CmsLSOpts
154 //
155 enum CmsLSOpts {LS_NULL=0, LS_IPO=0x0100, LS_IDNT=0x0200,
156  LS_ANY =0x0400, LS_IFMASK = 0x0f};
157 
158 XrdCmsSelected *List(SMask_t mask, CmsLSOpts opts, bool &oksel);
159 
160 // Returns the location of a file
161 //
162 int Locate(XrdCmsSelect &Sel);
163 
164 // Always run as a separate thread to monitor subscribed node performance
165 //
166 void *MonPerf();
167 
168 // Alwats run as a separate thread to maintain the node reference count
169 //
170 void *MonRefs();
171 
172 // Return total number of redirect references
173 //
174 long long Refs() {return SelWtot+SelRtot;}
175 
176 // Called to remove a node from the cluster
177 //
178 void Remove(XrdCmsNode *theNode);
179 
180 void Remove(const char *reason, XrdCmsNode *theNode, int immed=0);
181 
182 // Called to reset the node reference counts for nodes matching smask
183 //
184 void ResetRef(SMask_t smask, bool isLocked=false);
185 
186 // Called to select the best possible node to serve a file (two forms)
187 //
188 static const int NotFound = -1; // Locate() failed to find resource
189 static const int Wait4CBk = -2; // Only returned by Locate()
190 static const int RetryErr = -3; // Used only by XrdCmsNode retry processing
191 static const int EReplete = -4; // Failed, error information is complete
192 
193 int Select(XrdCmsSelect &Sel);
194 
195 int Select(SMask_t pmask, int &port, char *hbuff, int &hlen,
196  int isrw, int isMulti, int ifWant);
197 
198 // Manipulate the global selection lock
199 //
200 void SLock(bool dolock, bool wrmode=true)
201  {if (dolock)
202  {if (wrmode) STMutex.WriteLock();
203  else STMutex.ReadLock();
204  }
205  else STMutex.UnLock();
206  }
207 
208 // Called to get cluster space (for managers and supervisors only)
209 //
210 void Space(XrdCms::SpaceData &sData, SMask_t smask);
211 
212 // Called to return statistics
213 //
214 int Stats(char *bfr, int bln); // Server
215 int Statt(char *bfr, int bln); // Manager
216 
217  XrdCmsCluster();
218 virtual ~XrdCmsCluster() {} // This object should never be deleted
219 
220 private:
221 XrdCmsNode *AddAlt(XrdCmsClustID *cidP, XrdLink *lp, int port, int Status,
222  int sport, const char *theNID, const char *theIF);
224 int Drop(int sent, int sinst, XrdCmsDrop *djp=0);
225 void Record(char *path, const char *reason, bool force=false);
226 bool maxBits(SMask_t mVec, int mbits);
227 int Multiple(SMask_t mVec);
228 enum {eExists, eDups, eROfs, eNoRep, eNoSel, eNoEnt}; // Passed to SelFail
229 int SelFail(XrdCmsSelect &Sel, int rc);
230 int SelNode(XrdCmsSelect &Sel, SMask_t pmask, SMask_t amask);
234 int SelDFS(XrdCmsSelect &Sel, SMask_t amask,
235  SMask_t &pmask, SMask_t &smask, int isRW);
236 void sendAList(XrdLink *lp);
237 void setAltMan(int snum, XrdLink *lp, int port);
238 int Unreachable(XrdCmsSelect &Sel, bool none);
239 int Unuseable(XrdCmsSelect &Sel);
240 
241 // Number of <host>:Port characters per entry was INET6_ADDRSTRLEN+10
242 //
243 static const int AltSize = 254; // We may revert to IP address
244 
245 XrdSysRWLock STMutex; // Protects all node information variables
246 XrdCmsNode *NodeTab[STMax]; // Current set of nodes
247 
248 int STHi; // NodeTab high watermark
250 RAtomic_llong SelWtot; // Total number of r/w selections (successful)
251 RAtomic_llong SelRtot; // Total number of r/o selections (successful)
252 RAtomic_llong SelTcnt; // Total number of all selections
253 
254 // The following is a list of IP:Port tokens that identify supervisor nodes.
255 // The information is sent via the try request to redirect nodes; as needed.
256 // The list is alays rotated by one entry each time it is sent.
257 //
258 char AltMans[STMax*AltSize]; // ||123.123.123.123:12345|| = 21
259 char *AltMend;
261 
262 // The following two variables are protected by the STMutex
263 //
264 SMask_t peerHost; // Nodes that are acting as peers
265 SMask_t peerMask; // Always ~peerHost
266 };
267 
269 
270 namespace XrdCms
271 {
272 extern XrdCmsCluster Cluster;
273 }
274 #endif
long long TotFr
Definition: XrdCmsCluster.hh:87
Definition: XrdCmsCluster.hh:155
XrdSysRWLock STMutex
Definition: XrdCmsCluster.hh:245
int AltMent
Definition: XrdCmsCluster.hh:260
int sFree
Definition: XrdCmsCluster.hh:92
Definition: XrdSysPthread.hh:329
Definition: XrdCmsCluster.hh:114
static const int NotFound
Definition: XrdCmsCluster.hh:188
int SelNode(XrdCmsSelect &Sel, SMask_t pmask, SMask_t amask)
Definition: XrdCmsClustID.hh:40
static const int CMS_Perm
Definition: XrdCmsCluster.hh:67
Definition: XrdCmsSelect.hh:104
int SelDFS(XrdCmsSelect &Sel, SMask_t amask, SMask_t &pmask, SMask_t &smask, int isRW)
Definition: XrdCmsCluster.hh:228
XrdCmsNode * NodeTab[STMax]
Definition: XrdCmsCluster.hh:246
SpaceData()
Definition: XrdCmsCluster.hh:96
Definition: XrdCmsCluster.hh:228
RAtomic_llong SelWtot
Definition: XrdCmsCluster.hh:250
Definition: YProtocol.hh:82
Definition: XrdCmsSelect.hh:141
XrdCmsNode * SelbyCost(SMask_t, XrdCmsSelector &selR)
Definition: XrdCmsCluster.hh:228
static const int Wait4CBk
Definition: XrdCmsCluster.hh:189
CmsLSOpts
Definition: XrdCmsCluster.hh:155
void ReadLock()
Definition: XrdSysPthread.hh:342
static const int RetryErr
Definition: XrdCmsCluster.hh:190
Definition: XrdCmsBaseFS.hh:48
RAtomic_llong SelRtot
Definition: XrdCmsCluster.hh:251
void Space(XrdCms::SpaceData &sData, SMask_t smask)
static const int EReplete
Definition: XrdCmsCluster.hh:191
virtual void BlackList(XrdOucTList *blP)
static const int CMS_hasAlts
Definition: XrdCmsCluster.hh:78
SMask_t peerHost
Definition: XrdCmsCluster.hh:264
Definition: XrdNetAddr.hh:41
XrdCmsNode * AddAlt(XrdCmsClustID *cidP, XrdLink *lp, int port, int Status, int sport, const char *theNID, const char *theIF)
Definition: XrdCmsCluster.hh:155
static struct XrdCl::None none
long long Total
Definition: XrdCmsCluster.hh:86
~SpaceData()
Definition: XrdCmsCluster.hh:99
void SLock(bool dolock, bool wrmode=true)
Definition: XrdCmsCluster.hh:200
static const int CMS_isPeer
Definition: XrdCmsCluster.hh:70
int sNum
Definition: XrdCmsCluster.hh:93
int NodeCnt
Definition: XrdCmsCluster.hh:119
RAtomic_llong SelTcnt
Definition: XrdCmsCluster.hh:252
void WriteLock()
Definition: XrdSysPthread.hh:343
int Unreachable(XrdCmsSelect &Sel, bool none)
int Select(XrdCmsSelect &Sel)
virtual ~XrdCmsCluster()
Definition: XrdCmsCluster.hh:218
int SelFail(XrdCmsSelect &Sel, int rc)
XrdCmsNode * Add(XrdLink *lp, int dport, int Status, int sport, const char *theNID, const char *theIF)
static const int CMS_noStage
Definition: XrdCmsCluster.hh:65
int wFree
Definition: XrdCmsCluster.hh:89
int Stats(char *bfr, int bln)
XrdCmsNode * SelbyRef(SMask_t, XrdCmsSelector &selR)
Definition: XrdCmsCluster.hh:155
static const int CMS_isVers3
Definition: XrdCmsCluster.hh:75
static const int CMS_isProxy
Definition: XrdCmsCluster.hh:71
void Record(char *path, const char *reason, bool force=false)
int Multiple(SMask_t mVec)
unsigned long long SMask_t
Definition: XrdCmsTypes.hh:33
SMask_t Broadcast(SMask_t, const struct iovec *, int, int tot=0)
SMask_t getMask(const XrdNetAddr *addr)
XrdCmsSelected * List(SMask_t mask, CmsLSOpts opts, bool &oksel)
char * AltMend
Definition: XrdCmsCluster.hh:259
Definition: XrdCmsCluster.hh:82
static const int CMS_isSuper
Definition: XrdCmsCluster.hh:73
#define STMax
Definition: XrdCmsTypes.hh:39
int Statt(char *bfr, int bln)
void * MonRefs()
int sUtil
Definition: XrdCmsCluster.hh:94
int Reserved
Definition: XrdCmsCluster.hh:249
Definition: XrdOucTList.hh:41
Definition: XrdCmsCluster.hh:156
int STHi
Definition: XrdCmsCluster.hh:248
void ResetRef(SMask_t smask, bool isLocked=false)
Definition: XrdCmsCluster.hh:156
XrdCmsCluster Cluster
static const int CMS_isMan
Definition: XrdCmsCluster.hh:68
SMask_t peerMask
Definition: XrdCmsCluster.hh:265
static const int CMS_Suspend
Definition: XrdCmsCluster.hh:66
int wNum
Definition: XrdCmsCluster.hh:90
void * MonPerf()
#define XRDOUC_ENUM_OPERATORS(T)
Definition: XrdOucEnum.hh:22
int Broadsend(SMask_t smask, XrdCms::CmsRRHdr &Hdr, void *Data, int Dlen)
static const int CMS_noSpace
Definition: XrdCmsCluster.hh:72
Definition: XrdCmsCluster.hh:228
XrdCmsNode * calcDelay(XrdCmsSelector &selR)
int Locate(XrdCmsSelect &Sel)
friend class XrdCmsDrop
Definition: XrdCmsCluster.hh:117
Definition: XrdCmsCluster.hh:228
int Drop(int sent, int sinst, XrdCmsDrop *djp=0)
Definition: XrdCmsNode.hh:57
bool maxBits(SMask_t mVec, int mbits)
int wUtil
Definition: XrdCmsCluster.hh:91
void setAltMan(int snum, XrdLink *lp, int port)
static const int CMS_Lost
Definition: XrdCmsCluster.hh:69
int wMinF
Definition: XrdCmsCluster.hh:88
char AltMans[STMax *AltSize]
Definition: XrdCmsCluster.hh:258
void sendAList(XrdLink *lp)
Definition: XrdCmsSelect.hh:43
static const int AltSize
Definition: XrdCmsCluster.hh:243
void UnLock()
Definition: XrdSysPthread.hh:348
static const int CMS_notServ
Definition: XrdCmsCluster.hh:77
long long Refs()
Definition: XrdCmsCluster.hh:174
XrdCmsNode * SelbyLoad(SMask_t, XrdCmsSelector &selR)
Definition: XrdCmsCluster.hh:228
void Remove(XrdCmsNode *theNode)
int Unuseable(XrdCmsSelect &Sel)