Remove duplicate string in an array

var arr_transcript = [
    "Sometext|22621086|address|333629dc87894a7ea7df5291fa6d1836|PC_E|1803",
    "Sometext2|22622138|working|d3e70175ffe942568cd21f1cf96f4d63|PC_E|1803",
    "Sometext3|22622138|working|851946e6325445da99c113951590f714|PC_E|1803"
];

var map = arr_transcript.map(function(text) { 
    return text.split('|')[1];
});

var filtered = arr_transcript.filter(function(item, index) {
    return index === map.lastIndexOf( map[index] );
});

console.log(filtered)

public class RemDup
{
        public static void main ( String[] args )
        {
            String sentence = "this is java programming program progress";
            int max_word_length = sentence.length()/2;
            int min_word_length = 2;
            while(max_word_length>=min_word_length)
            {
            int si = 0;
            int ei = max_word_length;
            while ( ei<sentence.length() )
            {
                int e=ei;
                while ( e<sentence.length() )
                {
                    int ind = sentence.indexOf ( sentence.substring ( si, ei ),e );
                    if ( ind!=-1 )
                    {
                        if(
                         sentence.substring(ind-1,ind).equals(" ")
                        &((ind+max_word_length)>=sentence.length()||
                        sentence.substring(ind+max_word_length,ind+max_word_length+1).equals(" "))
                        )
                        {
                        sentence = sentence.substring ( 0,ind ) +sentence.substring ( ind+max_word_length,sentence.length() );
                        }
                        e=ind+max_word_length;

                    }
                    else break;
                }


                si+=1;
                ei+=1;

            }
            max_word_length--;
            }
            System.out.println(sentence);
        }

}

val removeDup = udf((str: String) => {
  str.split("\\(|\\)").filter(s => s != "," && s != "").map(s => {
    val array = s.replace("(", "").replace(")", "").split(",")
    (array(0), array(1))
  })
  .groupBy(_._1)
  .mapValues(a => a.sortBy(_._2).head)
  .values
  .toSeq
  .sortBy(_._1)
})

val df = spark.sparkContext.parallelize(Seq("(1,2),(1,3),(1,4),(2,3)").toDF("colA")

df.select(removeDup('colA)).show

+--------------------+
|           UDF(colA)|
+--------------------+
|      [[1,2], [2,3]]|
+--------------------+

string[] newArray = myString.SelectMany(s => s.Split(' ')).Distinct().ToArray();

string[] newArray = myString.SelectMany(s => s.Split(' ')).Distinct(StringComparer.InvariantCultureIgnoreCase).ToArray();