DOC/MGM/CONSOLE: Update documentation on the Converter functionality,

remove unused CLI options like convert rule, update the statistics
counters to reflect the new values being collected and update the
convert CLI help message.

Author: esindril

console/commands/com_proto_convert.cc

@@ -135,21 +135,6 @@ ConvertHelper::ParseCommand(const char* arg)
 
     file ->set_allocated_conversion(conversion);
     // Placeholder for options
-  } else if (token == "rule") {
-    if (!tokenizer.NextToken(token)) {
-      return false;
-    }
-
-    eos::console::ConvertProto_RuleProto* rule = convert->mutable_rule();
-    rule->set_allocated_identifier(ParseIdentifier(token));
-    eos::console::ConvertProto_ConversionProto*
-    conversion = ParseConversion(tokenizer);
-
-    if (conversion == nullptr) {
-      return false;
-    }
-
-    rule->set_allocated_conversion(conversion);
   } else if (token == "list") {
     convert->mutable_list();
   } else if (token == "clear") {
@@ -319,11 +304,6 @@ void com_convert_help()
       << "  convert file <identifier> <conversion>           \n"
       << "    schedule a file conversion                     \n"
       << "    <identifier> = fid|fxid|path                   \n"
-      << "    <conversion> = <layout:replica> [space] [placement] [checksum]\n"
-      << std::endl
-      << "  convert rule <identifier> <conversion>           \n"
-      << "    apply a conversion rule on the given directory \n"
-      << "    <identifier> = cid|cxid|path                   \n"
       << "    <conversion> = <layout:replica> [space] [placement] [checksum]\n";
   std::cerr << oss.str() << std::endl;
 }

doc/diopside/manual/microservices.rst

@@ -860,7 +860,7 @@ LongLongAggregator::aggregateNodes(
 //----------------------------------------------------------------------------
 FsSpace::FsSpace(const char* name)
   : BaseView(common::SharedHashLocator::makeForSpace(name)),
-    mFsBalancer(nullptr), mConverter(nullptr), mGroupBalancer(nullptr),
+    mFsBalancer(nullptr), mGroupBalancer(nullptr),
     mGeoBalancer(nullptr), mGroupDrainer(nullptr)
 {
   mName = name;
@@ -1012,12 +1012,12 @@ FsSpace::FsSpace(const char* name)
 
     // Disable lru by default
     if (GetConfigMember("lru").empty()) {
-      SetConfigMember("converter", "off");
+      SetConfigMember("lru", "off");
     }
 
     // Set one week lru interval by default
     if (GetConfigMember("lru.interval") == "604800") {
-      SetConfigMember("converter.ntx", "2");
+      SetConfigMember("lru.interval", "604800");
     }
 
     // Set the wfe off by default
@@ -1757,9 +1757,6 @@ FsView::GetSpaceFormat(std::string option)
     format += "member=cfg.quota:width=6:format=s|";
     format += "member=cfg.balancer:width=10:format=s:tag=balancing|";
     format += "member=cfg.balancer.threshold:width=11:format=+l:tag=threshold|";
-    format += "member=cfg.converter:width=11:format=s:tag=converter|";
-    format += "member=cfg.converter.ntx:width=6:format=+l:tag=ntx|";
-    format += "member=cfg.stat.converter.active:width=8:format=+l:tag=active|";
     format += "member=cfg.wfe:width=11:format=s:tag=wfe|";
     format += "member=cfg.wfe.ntx:width=6:format=+l:tag=ntx|";
     format += "member=cfg.stat.wfe.active:width=8:format=+l:tag=active|";

mgm/FsView.cc

@@ -572,8 +572,6 @@ public:
   static std::string gConfigQueuePrefix; ///<  Configuration queue prefix
   //! File system balancer
   std::unique_ptr<FsBalancer> mFsBalancer;
-  //! Threaded object running layout conversion jobs
-  Converter* mConverter;
   //! Threaded object running group balancing
   GroupBalancer* mGroupBalancer;
   //! Threaded object running geotag balancing

mgm/FsView.hh

@@ -1132,14 +1132,15 @@ XrdMgmOfs::Configure(XrdSysError& Eroute)
           if (val != nullptr) {
             JwtTokenPath = val;
           }
+
           Eroute.Say("=====> mgmofs.jwttokenpath : ", val);
         }
       }
 
       if (!strcmp("protowfusegrpctls", var)) {
         if ((!(val = Config.GetWord())) ||
             (strcmp("true", val) && strcmp("false", val) &&
-              strcmp("1", val) && strcmp("0", val))) {
+             strcmp("1", val) && strcmp("0", val))) {
           Eroute.Emsg("Config", "argument for protowfusegrpctls is invalid. "
                       "Must be <true>, <false>, <1> or <0>!");
         } else {
@@ -1231,6 +1232,7 @@ XrdMgmOfs::Configure(XrdSysError& Eroute)
   } else {
     Eroute.Say("=====> mgmofs.protowfusegrpc : false");
   }
+
   if (protowfusegrpctls) {
     Eroute.Say("=====> mgmofs.protowfusegrpctls : true");
   } else {
@@ -2389,8 +2391,9 @@ XrdMgmOfs::InitStats()
   MgmStats.Add("CommitFailedNamespace", 0, 0, 0);
   MgmStats.Add("CommitFailedParameters", 0, 0, 0);
   MgmStats.Add("CommitFailedUnlinked", 0, 0, 0);
-  MgmStats.Add("ConversionDone", 0, 0, 0);
-  MgmStats.Add("ConversionFailed", 0, 0, 0);
+  MgmStats.Add("ConversionJobStared", 0, 0, 0);
+  MgmStats.Add("ConversionJobFailed", 0, 0, 0);
+  MgmStats.Add("ConversionJobSuccessful", 0, 0, 0);
   MgmStats.Add("CopyStripe", 0, 0, 0);
   MgmStats.Add("Devices::Extract", 0, 0, 0);
   MgmStats.Add("Devices::Store", 0, 0, 0);

mgm/XrdMgmOfsConfigure.cc

@@ -68,8 +68,6 @@ ConvertCmd::ProcessRequest() noexcept
     ConfigSubcmd(convert.config(), reply, json);
   } else if (subcmd == eos::console::ConvertProto::kFile) {
     FileSubcmd(convert.file(), reply, json);
-  } else if (subcmd == eos::console::ConvertProto::kRule) {
-    RuleSubcmd(convert.rule(), reply, json);
   } else if (subcmd == eos::console::ConvertProto::kList) {
     ListSubcmd(convert.list(), reply, json);
   } else if (subcmd == eos::console::ConvertProto::kClear) {
@@ -302,97 +300,6 @@ void ConvertCmd::FileSubcmd(const eos::console::ConvertProto_FileProto& file,
   reply.set_std_out(out.str());
 }
 
-//------------------------------------------------------------------------------
-// Execute rule subcommand
-//------------------------------------------------------------------------------
-void ConvertCmd::RuleSubcmd(const eos::console::ConvertProto_RuleProto& rule,
-                            eos::console::ReplyProto& reply,
-                            bool json)
-{
-  using eos::common::LayoutId;
-  auto conversion = rule.conversion();
-  XrdOucErrInfo errInfo;
-  std::ostringstream out;
-  std::ostringstream err;
-  std::string errmsg;
-  std::string path;
-  int retc = 0;
-  auto enforce_dir = XrdSfsFileExistence::XrdSfsFileExistIsDirectory;
-  path = PathFromIdentifierProto(rule.identifier(), errmsg);
-
-  if (!path.length()) {
-    reply.set_std_err(errmsg);
-    reply.set_retc(errno);
-    return;
-  }
-
-  if ((retc = CheckValidPath(path.c_str(), mVid, errmsg, enforce_dir))) {
-    reply.set_std_err(errmsg);
-    reply.set_retc(retc);
-    return;
-  }
-
-  if ((retc = CheckConversionProto(conversion, errmsg))) {
-    reply.set_std_err(errmsg);
-    reply.set_retc(retc);
-    return;
-  }
-
-  if (conversion.checksum().empty()) {
-    err << "error: no conversion checksum provided";
-    reply.set_std_err(err.str());
-    reply.set_retc(EINVAL);
-    return;
-  }
-
-  // Handle space default scenario
-  std::string space = conversion.space().empty() ?
-                      "default.0" : conversion.space();
-  // Handle checksum
-  LayoutId::eChecksum echecksum = static_cast<LayoutId::eChecksum>(
-                                    LayoutId::GetChecksumFromString(conversion.checksum()));
-  //------------------------------------------
-  // This part acts as a placeholder
-  //------------------------------------------
-  // Build conversion rule
-  std::string conversion_rule = BuildConversionId(conversion.layout(), echecksum,
-                                conversion.replica(), 0, space,
-                                conversion.placement());
-  size_t pos = conversion_rule.find(":");
-
-  if (pos != std::string::npos) {
-    conversion_rule.erase(0, pos + 1);
-  }
-
-  // Set rule as extended attribute
-  eos_info("msg=\"placing conversion rule\" path=%s conversion_rule=%s",
-           path.c_str(), conversion_rule.c_str());
-
-  if (gOFS->_attr_set(path.c_str(), errInfo, mVid, 0, "sys.eos.convert.rule",
-                      conversion_rule.c_str())) {
-    err << "error: could not set conversion rule '" << conversion_rule
-        << "' on path '" << path << "' -- emsg=" << errInfo.getErrText();
-    reply.set_std_err(err.str());
-    reply.set_retc(errInfo.getErrInfo());
-    return;
-  }
-
-  if (json) {
-    Json::Value json;
-    json["conversion_rule"] = conversion_rule;
-    json["path"] = path;
-    Json::StreamWriterBuilder builder;
-    std::unique_ptr<Json::StreamWriter> jsonwriter(
-      builder.newStreamWriter());
-    jsonwriter->write(json, &out);
-  } else {
-    out << "Set conversion rule '" << conversion_rule
-        << "' on path '" << path << "'";
-  }
-
-  reply.set_std_out(out.str());
-}
-
 //------------------------------------------------------------------------------
 // List jobs subcommand
 //------------------------------------------------------------------------------

mgm/proc/admin/ConvertCmd.cc

@@ -86,16 +86,6 @@ private:
   void FileSubcmd(const eos::console::ConvertProto_FileProto& file,
                   eos::console::ReplyProto& reply, bool json = false);
 
-  //----------------------------------------------------------------------------
-  //! Execute rule subcommand
-  //!
-  //! @param rule rule subcommand proto object
-  //! @param reply reply proto object
-  //! @param json flag to print output in JSON format
-  //----------------------------------------------------------------------------
-  void RuleSubcmd(const eos::console::ConvertProto_RuleProto& rule,
-                  eos::console::ReplyProto& reply, bool json = false);
-
   //----------------------------------------------------------------------------
   //! List jobs subcommand
   //!

mgm/proc/admin/ConvertCmd.hh

@@ -127,7 +127,6 @@ function disable_altxs_sync() {
   for fsid in $(eos -j fs ls | jq '.result[].id'); do
     eos fs config ${fsid} altxs_sync=0
     eos fs config ${fsid} altxs_sync_interval=0
-    # Put back teh default value
   done
 }